강의영상

import torch 
import matplotlib.pyplot as plt

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

plt.plot(X,y)

[<matplotlib.lines.Line2D at 0x7f6f577c6820>]

네트워크 설정, 옵티마이저, 로스

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=512,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

모형학습

for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad()

plt.plot(X,y) 
plt.plot(X,yhat.data)

[<matplotlib.lines.Line2D at 0x7f6f54ac3760>]

train / validation

X1=X[:80]
y1=y[:80]
X2=X[80:]
y2=y[80:]

torch.manual_seed(1) 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=512,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

for epoc in range(1000): 
    ## 1 
    y1hat=net(X1) 
    ## 2 
    loss=loss_fn(y1hat,y1) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step() 
    net.zero_grad()

plt.plot(X,y)
plt.plot(X1,net(X1).data,'--r') 
plt.plot(X2,net(X2).data,'--g')

[<matplotlib.lines.Line2D at 0x7f6f549d9790>]

드랍아웃

X1=X[:80]
y1=y[:80]
X2=X[80:]
y2=y[80:]

torch.manual_seed(1) 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Dropout(0.8),
    torch.nn.Linear(in_features=512,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

for epoc in range(1000): 
    ## 1 
    y1hat=net(X1) 
    ## 2 
    loss=loss_fn(y1hat,y1) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step() 
    net.zero_grad()

net.eval() ## 네트워크를 평가모드로 전환
plt.plot(X,y)
plt.plot(X1,net(X1).data,'--r') 
plt.plot(X2,net(X2).data,'--g')

[<matplotlib.lines.Line2D at 0x7f6f401b5a90>]

학습과정 비교 (주의: 코드복잡함)

- 데이터 생성

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1) 
y=torch.randn(100).reshape(100,1)

- tr/val 분리

X_tr=X[:80]
y_tr=y[:80]
X_val=X[80:]
y_val=y[80:]

- 네트워크, 옵티마이저, 손실함수 설정

드랍아웃을 이용한 네트워트 (net2)와 그렇지 않은 네트워크 (net1)
대응하는 옵티마이저 1,2 설정
손실함수

torch.manual_seed(1) 
net1=torch.nn.Sequential(
    torch.nn.Linear(1,512), 
    torch.nn.ReLU(),
    torch.nn.Linear(512,1)) 
optimizer_net1 = torch.optim.Adam(net1.parameters()) 
net2=torch.nn.Sequential(
    torch.nn.Linear(1,512), 
    torch.nn.ReLU(),
    torch.nn.Dropout(0.8),
    torch.nn.Linear(512,1)) 
optimizer_net2 = torch.optim.Adam(net2.parameters())
loss_fn=torch.nn.MSELoss()

tr_loss_net1=[] 
val_loss_net1=[]
tr_loss_net2=[] 
val_loss_net2=[]

for epoc in range(1000): 
    ## 1 
    yhat_tr_net1 = net1(X_tr) 
    ## 2 
    loss_tr = loss_fn(yhat_tr_net1, y_tr) 
    ## 3 
    loss_tr.backward() 
    ## 4 
    optimizer_net1.step()
    net1.zero_grad() 
    ## 5 기록 
    ### tr 
    tr_loss_net1.append(loss_tr.item())
    
    ### val 
    yhat_val_net1 = net1(X_val) 
    loss_val = loss_fn(yhat_val_net1,y_val) 
    val_loss_net1.append(loss_val.item())

for epoc in range(1000): 
    ## 1 
    yhat_tr_net2 = net2(X_tr) 
    ## 2 
    loss_tr = loss_fn(yhat_tr_net2, y_tr) 
    ## 3 
    loss_tr.backward() 
    ## 4 
    optimizer_net2.step()
    net2.zero_grad() 
    ## 5 기록 
    ### tr 
    net2.eval()
    tr_loss_net2.append(loss_tr.item())
    ### val 
    yhat_val_net2 = net2(X_val) 
    loss_val = loss_fn(yhat_val_net2,y_val) 
    val_loss_net2.append(loss_val.item())
    net2.train()

net2.eval() 
fig , ((ax1,ax2),(ax3,ax4)) = plt.subplots(2,2)
ax1.plot(X,y,'.');ax1.plot(X_tr,net1(X_tr).data); ax1.plot(X_val,net1(X_val).data); 
ax2.plot(X,y,'.');ax2.plot(X_tr,net2(X_tr).data); ax2.plot(X_val,net2(X_val).data); 
ax3.plot(tr_loss_net1);ax3.plot(val_loss_net1);
ax4.plot(tr_loss_net2);ax4.plot(val_loss_net2);

- 다 좋은데 코드를 짜는것이 너무 힘들다.

생각해보니까 미니배치도 만들어야 함 + 미니배치를 나눈상태에서 GPU 메모리에 파라메터도 올려야함.
조기종료와 같은 기능도 구현해야함 + 기타등등을 구현해야함.
나중에는 학습률을 서로 다르게 돌려가며 결과도 기록해야함 $\to$ 그래야 좋은 학습률 선택가능
for문안에 step1~step4를 넣는것도 너무 반복작업임.
등등..

- 위와 같은 것들의 특징: 머리로 상상하기는 쉽지만 실제 구현하는 것은 까다롭다.

- 사실 우리가 하고싶은것

아키텍처를 설계: 데이터를 보고 맞춰서 설계해야할 때가 많음 (우리가 해야한다)
손실함수: 통계학과 교수님들이 연구하심
옵티마이저: 산공교수님들이 연구하심

- 제 생각

기업의욕심: read-data를 분석하는 딥러닝 아키텍처 설계 $\to$ 아키텍처별로 결과를 관찰 (편하게) $\Longrightarrow$ fastai + read data
학생의욕심: 그러면서도 모형이 돌아가는 원리는 아주 세밀하게 알고싶음 $\Longrightarrow$ pytorch + toy example (regression 등을 위주로)
연구자의욕심: 기존의 모형을 조금 변경해서 쓰고싶음 $\Longrightarrow$ (pytorch +fastai) + any data

- tensorflow + keras vs pytorch + fastai

pytorch + fastai

- 데이터셋을 만든다.

X_tr=X[:80]
y_tr=y[:80]
X_val=X[80:]
y_val=y[80:]

ds1=torch.utils.data.TensorDataset(X_tr,y_tr) 
ds2=torch.utils.data.TensorDataset(X_val,y_val)

- 데이터로더를 만든다.

dl1 = torch.utils.data.DataLoader(ds1, batch_size=80) 
dl2 = torch.utils.data.DataLoader(ds2, batch_size=20)

- 데이터로더스를 만든다.

from fastai.vision.all import *

dls=DataLoaders(dl1,dl2)

드랍아웃 제외버전

- 네트워크 설계 (드랍아웃 제외)

torch.manual_seed(1) 
net_fastai = torch.nn.Sequential(
    torch.nn.Linear(in_features=1, out_features=512),
    torch.nn.ReLU(),
    #torch.nn.Dropout(0.8),
    torch.nn.Linear(in_features=512, out_features=1)) 
#optimizer 
loss_fn=torch.nn.MSELoss()

- 러너오브젝트 (for문 대신돌려주는 오브젝트)

lrnr= Learner(dls,net_fastai,opt_func=Adam,loss_func=loss_fn)

- 에폭만 설정하고 바로 학습

lrnr.fit(1000)

- loss들도 에폭별로 기록되어 있음

lrnr.recorder.plot_loss()

- net_fastai에도 파라메터가 업데이트 되어있음

# list(net_fastai.parameters())

리스트를 확인해보면 net_fastai 의 파라메터가 알아서 GPU로 옮겨져서 학습됨.

- 플랏

net_fastai.to("cpu") 
plt.plot(X,y,'.')
plt.plot(X_tr,net_fastai(X_tr).data) 
plt.plot(X_val,net_fastai(X_val).data)

[<matplotlib.lines.Line2D at 0x7f6e97599790>]

드랍아웃 추가버전

- 네트워크 설계 (드랍아웃 추가)

torch.manual_seed(1) 
net_fastai = torch.nn.Sequential(
    torch.nn.Linear(in_features=1, out_features=512),
    torch.nn.ReLU(),
    torch.nn.Dropout(0.8),
    torch.nn.Linear(in_features=512, out_features=1)) 
#optimizer 
loss_fn=torch.nn.MSELoss()

- 러너오브젝트 (for문 대신돌려주는 오브젝트)

lrnr= Learner(dls,net_fastai,opt_func=Adam,loss_func=loss_fn)

- 에폭만 설정하고 바로 학습

lrnr.fit(1000)

- loss들도 에폭별로 기록되어 있음

lrnr.recorder.plot_loss()

- net_fastai에도 파라메터가 업데이트 되어있음

리스트를 확인해보면 net_fastai 의 파라메터가 알아서 GPU로 옮겨져서 학습됨.

- 플랏

net_fastai.to("cpu") 
plt.plot(X,y,'.')
plt.plot(X_tr,net_fastai(X_tr).data) 
plt.plot(X_val,net_fastai(X_val).data)

[<matplotlib.lines.Line2D at 0x7f6e94e31640>]

CPU vs GPU 시간비교

import time

CPU (512)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=512,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

0.6667273044586182

GPU (512)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=512,out_features=1)) 
net.to("cuda:0")
X=X.to("cuda:0")
y=y.to("cuda:0")
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

2.074880838394165

- ?? CPU가 더 빠르다!!

CPU (20480)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=20480), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=20480,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

3.695246696472168

GPU (20480)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=20480), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=20480,out_features=1)) 
net.to("cuda:0")
X=X.to("cuda:0")
y=y.to("cuda:0")
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

2.2188520431518555

CPU (204800)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=204800), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=204800,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

62.97744035720825

GPU (204800)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=204800), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=204800,out_features=1)) 
net.to("cuda:0")
X=X.to("cuda:0")
y=y.to("cuda:0")
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

2.404008626937866

숙제

- 현재 작업하고 있는 컴퓨터에서 아래코드를 실행후 시간을 출력하여 스샷제출

CPU (512)

torch.manual_seed(5) 
X=torch.linspace(0,1,100).reshape(100,1)
y=torch.randn(100).reshape(100,1)*0.01

torch.manual_seed(1) # 초기가중치를 똑같이 
net=torch.nn.Sequential(
    torch.nn.Linear(in_features=1,out_features=512), 
    torch.nn.ReLU(),
    torch.nn.Linear(in_features=512,out_features=1)) 
optimizer= torch.optim.Adam(net.parameters())
loss_fn= torch.nn.MSELoss()

t1=time.time()
for epoc in range(1000): 
    ## 1 
    yhat=net(X) 
    ## 2 
    loss=loss_fn(yhat,y) 
    ## 3 
    loss.backward()
    ## 4 
    optimizer.step()
    net.zero_grad() 
t2=time.time()

t2-t1

0.6667273044586182

epoch	train_loss	valid_loss	time
0	1.277156	0.491314	00:00
1	1.277145	0.455286	00:00
2	1.275104	0.444275	00:00
3	1.274429	0.465787	00:00
4	1.273436	0.507203	00:00
5	1.272421	0.548102	00:00
6	1.271840	0.561292	00:00
7	1.271377	0.549409	00:00
8	1.270855	0.530416	00:00
9	1.270437	0.520700	00:00
10	1.270176	0.526273	00:00
11	1.269935	0.543579	00:00
12	1.269655	0.562939	00:00
13	1.269411	0.571586	00:00
14	1.269217	0.563700	00:00
15	1.269018	0.543646	00:00
16	1.268787	0.521385	00:00
17	1.268563	0.505799	00:00
18	1.268362	0.500011	00:00
19	1.268159	0.501830	00:00
20	1.267941	0.506255	00:00
21	1.267730	0.506739	00:00
22	1.267540	0.499733	00:00
23	1.267353	0.487385	00:00
24	1.267163	0.474839	00:00
25	1.266981	0.466926	00:00
26	1.266814	0.465347	00:00
27	1.266648	0.468656	00:00
28	1.266480	0.473641	00:00
29	1.266316	0.476266	00:00
30	1.266156	0.474677	00:00
31	1.265996	0.469958	00:00
32	1.265833	0.465630	00:00
33	1.265673	0.464544	00:00
34	1.265514	0.467181	00:00
35	1.265355	0.472571	00:00
36	1.265194	0.477105	00:00
37	1.265037	0.478357	00:00
38	1.264880	0.475766	00:00
39	1.264724	0.471696	00:00

epoch	train_loss	valid_loss	time
0	1.585653	0.428918	00:00
1	1.552326	0.434847	00:00
2	1.568810	0.442775	00:00
3	1.543528	0.449585	00:00
4	1.562597	0.456666	00:00
5	1.523623	0.459943	00:00
6	1.506816	0.458130	00:00
7	1.510407	0.455353	00:00
8	1.532602	0.449054	00:00
9	1.528153	0.445443	00:00
10	1.518390	0.442207	00:00
11	1.508012	0.442086	00:00
12	1.498026	0.443293	00:00
13	1.502874	0.444508	00:00
14	1.502828	0.445713	00:00
15	1.496831	0.446047	00:00
16	1.483070	0.447462	00:00
17	1.496551	0.449803	00:00
18	1.482904	0.450663	00:00
19	1.471269	0.453689	00:00
20	1.467480	0.456816	00:00
21	1.457825	0.460537	00:00
22	1.450724	0.463197	00:00
23	1.445010	0.466199	00:00
24	1.441184	0.471516	00:00
25	1.436977	0.474600	00:00
26	1.431098	0.476256	00:00
27	1.423327	0.478671	00:00
28	1.416092	0.479825	00:00
29	1.414993	0.478338	00:00
30	1.421260	0.477377	00:00
31	1.413346	0.474661	00:00
32	1.417670	0.470384	00:00
33	1.412011	0.468277	00:00
34	1.414570	0.465151	00:00
35	1.416442	0.461778	00:00
36	1.410454	0.457763	00:00
37	1.405844	0.453920	00:00
38	1.405701	0.451884	00:00
39	1.405358	0.450063	00:00