2017年10月29日日曜日

MNISTでのmixup法の効果

数日前にarXivに投稿されていたmixup法(https://arxiv.org/abs/1710.09412)をMNISTで試してみました。 手法自体は簡単で、ニューラルネットワークの学習時に使う入力と出力を2つずつランダムに選び、それを比率\(\lambda\)で混ぜるだけです。式で書くと、ランダムに選んだ入力と出力を\((x_i,y_i), (x_j,y_j)\)のように2つ選ぶとき、学習時に渡す入力\(\tilde{x}\)と出力\(\tilde{y}\)はそれぞれ、 \[ \tilde{x} = \lambda x_i + (1-\lambda) x_j \] \[ \tilde{y} = \lambda y_i + (1-\lambda) y_j \] となります。\(\lambda\)はサンプルごとにベータ分布からランダムに決めます。

今回はMNISTで試すので、数字が半透明で重なります。ベータ分布の\(\alpha=\beta=2\)で32個生成した結果が下図です。

\(\alpha=\beta\)の下で、\(\alpha\)を色々変えて学習した結果が下表です。学習は30エポックのみです。

\(\alpha\) Train acc Validation acc
n/a 0.999933 0.991600
0.1 0.999767 0.991800
0.2 0.999483 0.990600
0.3 0.999283 0.990300
0.4 0.998983 0.991600
0.5 0.998633 0.989500
1 0.997483 0.989700
2 0.996383 0.990000

1行目は通常の方法で混ぜずに学習させた場合の結果です。Train accの計算には混ぜていない学習用データを用いています。\(\alpha\)を増やすと半々で混ざるサンプルの確率が増え、Train accが徐々に下がっていきます。一方、Validation accは下がっているようにも見えますが、\(\alpha=0.1\)の場合は若干上がっています。差が小さすぎるので誤差かもしれません。

論文に書かれているような効果は今回のMNIST実験では見られませんでしたが、こういう混ぜ方をしても学習ができることは確認できました。

以下、今回の実験で使ったコードです。122行目のeh.alphaの値を変えることで\(\alpha\)を変更します。

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# -*- coding: utf-8 -*-
import sys,os,math
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../keras-examples/src')
import numpy as np
import keras
from keras.models import Sequential, Model, Input, model_from_json
from keras.layers import Dense, Activation, Reshape, Flatten, Dropout
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Conv2D
from keras.layers.advanced_activations import LeakyReLU
from keras.utils import to_categorical
from util.history import ExperimentHistory
from keras.datasets import mnist
from PIL import Image, ImageDraw, ImageFont

TRAIN_EPOCH = 30
GENERATED_IMAGE_PATH = 'generated_images/'
FONT_DIR = "C:/Windows/Fonts"

# Image size is the same as MNIST
IMAGE_WIDTH = 28
IMAGE_HEIGHT = 28

def make_model():
    model = Sequential()
    model.add(Conv2D(64, (5, 5), strides=(2, 2), padding='same',
                     input_shape=(IMAGE_WIDTH, IMAGE_HEIGHT, 1), 
                     data_format='channels_last'))
    model.add(LeakyReLU(0.2))
    model.add(Conv2D(128, (5, 5), strides=(2, 2), data_format='channels_last'))
    model.add(LeakyReLU(0.2))
    model.add(Flatten())
    model.add(Dense(256))
    model.add(LeakyReLU(0.2))
    model.add(Dropout(0.5))
    model.add(Dense(10))
    model.add(Activation('softmax'))
    print(model.summary())
    return model

def combine_images(generated_images, margin=20):
    total = len(generated_images)
    cols = int(math.sqrt(total))
    rows = math.ceil(float(total)/cols)
    width, height = generated_images[0].shape[0:2]
    print(generated_images[0].shape)
    combined_image = np.ones(((height+margin)*rows, width*cols), dtype=generated_images[0].dtype)*-0.5
    for index, image in enumerate(generated_images):
        i = int(index/cols)
        j = index % cols
        combined_image[(height+margin)*i:(height+margin)*i+height, width*j:width*(j+1)] = image[:, :, 0]
    return combined_image, cols, rows, width, height, margin

# Data generator
def data_gen(x, y, batch_size, alpha, num_classes):
    # Generate data eternally
    assert len(x) == len(y)
    epoch = 0
    index = 0
    font = ImageFont.truetype(FONT_DIR+"/{}".format("Tahoma.ttf"), 10) # For drawing numbers
    while True:
        index += 1
        data = []
        labels = []
        for i in range(batch_size):
            i1 = np.random.randint(0, len(x)) 
            i2 = np.random.randint(0, len(x)) 
            lam = np.random.beta(alpha, alpha)
            # Mixup two samples
            xm = lam * x[i1] + (1.0 - lam) * x[i2]
            ym = lam * y[i1] + (1.0 - lam) * y[i2]
            data.append(xm)
            labels.append(ym)

        ## Draw mixed-up images
        # image, cols, rows, width, height, margin = combine_images(data)
        # image = image*127.5 + 127.5
        # if not os.path.exists(GENERATED_IMAGE_PATH):
        #     os.mkdir(GENERATED_IMAGE_PATH)
        # img = Image.fromarray(image.astype(np.uint8))
        # draw = ImageDraw.Draw(img)
        # for i in range(len(data)):
        #     yi = int(i/cols)
        #     xi = i % cols
        # img.save(GENERATED_IMAGE_PATH+"%04d_%04d.png" % (epoch, index))

        yield np.array(data), np.array(labels).reshape((batch_size, num_classes))

def run(eh):
    # Prepare MNIST data
    (x_train, y_train), (x_val, y_val) = mnist.load_data()
    x_val = (x_val.astype(np.float32) - 127.5)/127.5
    x_val = x_val.reshape(x_val.shape[0], x_val.shape[1], x_val.shape[2], 1)
    y_val = keras.utils.to_categorical(y_val, num_classes=10)

    x_train = (x_train.astype(np.float32) - 127.5)/127.5
    x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1)
    y_train = keras.utils.to_categorical(y_train, num_classes=10)

    # Make a model
    opt = keras.optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999,
                                  epsilon=1e-08, decay=1e-4)
    model = make_model()
    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])

    # Train the model
    dg = data_gen(x_train, y_train, eh.batch_size, eh.alpha, 10)
    hist = model.fit_generator(dg, steps_per_epoch=len(x_train)/eh.batch_size, epochs=TRAIN_EPOCH,
                               validation_data=(x_val, y_val)).history

    # Evaluate the model by train and validation data
    score_train = model.evaluate(x_train, y_train, batch_size=eh.batch_size)
    score_val = model.evaluate(x_val, y_val, batch_size=eh.batch_size)

    comments = "#Final model loss_train={0:10.6f} acc_train={1:10.6f} loss_val={2:10.6f} acc_val={3:10.6f}"\
               .format(score_train[0], score_train[1], score_val[0], score_val[1])
    eh.write("history.log", model, opt, hist, comments)

if __name__ == '__main__':
    eh = ExperimentHistory()
    eh.batch_size = 32
    eh.alpha = 2
    run(eh)

0 件のコメント :