[딥러닝 스터디] MLP모델을 사용해 여러 옵티마이저의 성능 비교

import torch
import torchvision.models as models
import torch.nn as nn
import torch.optim as optim
from sklearn.datasets import fetch_openml
import matplotlib.pyplot as plt
import torch
from torch.utils.data import TensorDataset, DataLoader
from sklearn.model_selection import train_test_split

import torch
import torchvision
import torchvision.transforms as transforms
import seaborn as sns
import pandas as pd
import numpy as np
import random

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

batch_size = 64

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)

trainset.data = torch.Tensor(trainset.data).reshape(-1,3072) / 255.0
testset.data = torch.Tensor(testset.data).reshape(-1,3072) / 255.0

trainset.targets = np.array(trainset.targets)
testset.targets = np.array(testset.targets)

trainset.targets = torch.LongTensor(trainset.targets.astype("int"))
testset.targets = torch.LongTensor(testset.targets.astype("int"))

trainset_tensor = TensorDataset(trainset.data,trainset.targets)
testset_tensor = TensorDataset(testset.data,testset.targets)


trainloader = torch.utils.data.DataLoader(trainset_tensor, batch_size=batch_size,
                                          shuffle=True, num_workers=2)


testloader = torch.utils.data.DataLoader(testset_tensor, batch_size=batch_size,
                                         shuffle=False, num_workers=2)

mnist = fetch_openml('mnist_784', version=1, cache=True)

random.seed(156)
torch.manual_seed(156)
if device == 'cuda':
    torch.cuda.manual_seed_all(156)

X_normalize = mnist.data.values / 255.0
X = mnist.data.values
y = mnist.target.astype("int")

X_train,X_test,y_train,y_test = train_test_split(X_normalize,y,test_size=1/7,random_state=0,shuffle=True)
X_train = torch.Tensor(X_train)
X_test = torch.Tensor(X_test)
y_train = torch.LongTensor(y_train)
y_test = torch.LongTensor(y_test.values)

ds_train = TensorDataset(X_train,y_train)
ds_test = TensorDataset(X_test,y_test)

loader_train = DataLoader(ds_train,batch_size=64,shuffle=True,drop_last=True)
loader_test = DataLoader(ds_test,batch_size=64,shuffle=False)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class MLP(nn.Module):
  def __init__(self,input_size,hidden_size,output_size,batch,dropout,trainset):
    super().__init__()
    self.input_size = input_size
    self.hidden_size = hidden_size
    self.output_size = output_size
    self.batch = batch
    self.dropout = dropout
    self.trainset = trainset

    if trainset == "mnist":

      if (self.batch == False) & (self.dropout == False):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("fc2",nn.Linear(hidden_size,hidden_size))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("fc3",nn.Linear(hidden_size,10))
    
      elif (self.batch == True) & (self.dropout == False):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("batch1",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("fc2",nn.Linear(hidden_size,hidden_size))
        self.linear.add_module("batch2",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("fc3",nn.Linear(hidden_size,output_size))

      elif (self.batch == False) & (self.dropout == True):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("dropout1",nn.Dropout(0.2))
        self.linear.add_module("fc2",nn.Linear(hidden_size,hidden_size))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("dropout2",nn.Dropout(0.2))
        self.linear.add_module("fc3",nn.Linear(hidden_size,output_size))

      elif (self.batch == True) & (self.dropout == True):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("batch1",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("dropout1",nn.Dropout(0.2))
        self.linear.add_module("fc2",nn.Linear(hidden_size,hidden_size))
        self.linear.add_module("batch2",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("dropout2",nn.Dropout(0.2))
        self.linear.add_module("fc3",nn.Linear(hidden_size,output_size))
    
    elif trainset == "cifar10":

      if (self.batch == False) & (self.dropout == False):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("fc2",nn.Linear(hidden_size,512))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("fc3",nn.Linear(512,output_size))

      elif (self.batch == True) & (self.dropout == False):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("batch1",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("fc2",nn.Linear(hidden_size,512))
        self.linear.add_module("batch2",nn.BatchNorm1d(512))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("fc3",nn.Linear(512,output_size))

      elif (self.batch == False) & (self.dropout == True):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("dropout1",nn.Dropout(0.5))
        self.linear.add_module("fc2",nn.Linear(hidden_size,512))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("dropout2",nn.Dropout(0.5))
        self.linear.add_module("fc3",nn.Linear(512,output_size))

      elif (self.batch == True) & (self.dropout == True):
        self.linear = nn.Sequential()
        self.linear.add_module("fc1",nn.Linear(input_size,hidden_size))
        self.linear.add_module("batch1",nn.BatchNorm1d(hidden_size))
        self.linear.add_module("relu1",nn.ReLU())
        self.linear.add_module("dropout1",nn.Dropout(0.5))
        self.linear.add_module("fc2",nn.Linear(hidden_size,512))
        self.linear.add_module("batch2",nn.BatchNorm1d(512))
        self.linear.add_module("relu2",nn.ReLU())
        self.linear.add_module("dropout2",nn.Dropout(0.5))
        self.linear.add_module("fc3",nn.Linear(512,output_size))


  def forward(self,x):
    out = self.linear(x)
    return out

def init_weights(m):
  if type(m) == nn.Linear:
    torch.nn.init.kaiming_uniform(m.weight)
    m.bias.data.fill_(0.01)

model = MLP(input_size=784,hidden_size=100,output_size=10,batch=False,dropout=False,trainset="mnist").to(device)
model.apply(init_weights)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),lr=0.01)
epochs = 10

def train_fit(loader_train,epochs):
  
  model.train()

  loss = []
  
  for epoch in range(epochs):

    avg_cost= 0
    total_batch = len(loader_train)
    
    for data,targets in loader_train:
      data = data.to(device)
      targets = targets.to(device)
      
      outputs = model(data)
      cost = criterion(outputs,targets)
      avg_cost += cost / total_batch
      optimizer.zero_grad()
      cost.backward()
      optimizer.step()
    
    loss.append(avg_cost)
    print("Epoch {} 일때 loss 값 {} 입니다.".format(epoch+1,avg_cost))
  return loss

def train(loader_train):
  model.eval()
  correct = 0

  with torch.no_grad():
    for data,targets in loader_train:
      data = data.to(device)
      targets = targets.to(device)

      outputs = model(data)

      _,predicted = torch.max(outputs.data,1)
      correct += predicted.eq(targets.data.view_as(predicted)).sum()
  
  data_num = len(loader_train.dataset)
  print("\n학습 데이터에서 예측 정확도: {}/{} ({:.0f}%)\n".format(correct,data_num,100 * correct / data_num))

def test(loader_test):
  model.eval()
  correct = 0

  with torch.no_grad():
    for data,targets in loader_test:
      data = data.to(device)
      targets = targets.to(device)

      outputs = model(data)

      _,predicted = torch.max(outputs.data,1)
      correct += predicted.eq(targets.data.view_as(predicted)).sum()
  
  data_num = len(loader_test.dataset)
  print("\n테스트 데이터에서 예측 정확도: {}/{} ({:.0f}%)\n".format(correct,data_num,100 * correct / data_num))

옵티마이저에 따른 loss값 그래프