【第15回】簡易体積算出アルゴリズム 入出力に関する検討2

今回の概要

• 出力値を体積に変更した結果について
• 回帰値→one-hot vecに変更した結果について
• 入力に画像サイズの情報の付与（プログラム）
• アンサンブル学習の準備

出力値を体積に変更

まず、今回は高負荷なハイパーパラメータを設定しているため、メモリ不足による停止を防ぐためにソースコードの最適化を行いました。
ソースコードをVGG19_train1.pyに示します。
結果として、前回までより分散が少なく、平均loss = 0.004690358456224204と前回の検討より1桁低い損失を示しました。
出力値を体積にしたことにより、大きく精度が向上したと考えられます。

python:VGG19_train1.py
import keras
import numpy as np
import matplotlib.pyplot as plt
import os
import re
import csv
import gc
from sklearn.model_selection import KFold
import keras.backend as K
import optuna

from keras.utils import np_utils
from keras.models import Sequential, Model
from keras.layers import Input, Conv2D, MaxPooling2D, Dense, Dropout, Activation, Flatten
from keras.preprocessing.image import array_to_img, img_to_array, load_img
from keras.applications import ResNet50, VGG19
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras import optimizers
from sklearn.model_selection import train_test_split
from keras.backend import clear_session

import tensorflow as tf

batch_size = 8
nb_epochs = 1000
fold_num = 5

X = []
Y = []

with open('X_data.csv') as f:
    for row in csv.reader(f):
        X.append(row)

with open('Y_data.csv') as f:
    for row in csv.reader(f):
        Y.append(row)

X = np.array(X)
Y = np.array(Y)

print(X.shape)
print(Y.shape)

kf = KFold(n_splits=fold_num, shuffle=True)

def objective(trial):
    # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
    # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    # K.set_session(sess)

    #最適化するパラメータの設定

    #中間層１のユニット数
    hidden_neurons1 = int(trial.suggest_discrete_uniform("hidden_neurons1", 100, 5000, 100))

    hidden_neurons2 = int(trial.suggest_discrete_uniform("hidden_neurons2", 100, 5000, 100))

    # drop = trial.suggest_uniform("drop", 0.1, 0.5)

    out_neurons = 1

    #optimizer
    lr = trial.suggest_loguniform("lr", 1e-1, 1e-4)

    activation = trial.suggest_categorical("activation", ["linear", "sigmoid", "relu"])

    loss_all = []

    for train, test in kf.split(X):

        es = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')

        model = Sequential()
        model.add(Dense(hidden_neurons1, input_shape= (25088, )))
        model.add(Activation(activation))
        model.add(Dropout(0.5))
        model.add(Dense(hidden_neurons2))
        model.add(Activation(activation))
        model.add(Dropout(0.5))
        model.add(Dense(out_neurons))
        model.add(Activation("linear"))
        model.compile(loss="mean_squared_error", metrics = ['accuracy'], optimizer=optimizers.Adam(lr=lr))

        history = model.fit(X[train],
                           Y[train],
                           batch_size=batch_size,
                           epochs=nb_epochs,
                           verbose=1,
                           validation_data=(X[test], Y[test]),
                           shuffle=True,
                           callbacks=[es])

        #検証用データに対する正答率が最大となるハイパーパラメータを求める
        loss = history.history["val_loss"]
        loss.sort()
        loss_all.append(loss[0])
        print(" ")
        print("Loss: "+str(loss[0]))
        print(" ")

        clear_session()
        del model, history
        gc.collect()


    print(loss_all)
    print("%.2f%% (+/- %.2f%%)" % (np.mean(loss_all), np.std(loss_all)))

    return np.mean(loss_all)

study = optuna.create_study()
study.optimize(objective, n_trials=50)
print("End!!")
print("All trials")
print(study.trials)
print("Best Parameters")
print(study.best_params)
print("Best Value")
print(study.best_value)

print("end of script")

回帰値→one-hot vecに変更

次に、出力にone-hot vecを導入して出力層を200とし、カテゴリカルに判別しました。
ソースコードをVGG19_train2.pyに示します。
その結果はloss = 0.0020341352004761572と上記の半分の損失のため、向上していることが分かります。
この変化により、精度が向上することが考えられます。

python:VGG19_train2.py
import keras
import numpy as np
import matplotlib.pyplot as plt
import os
import re
import csv
import gc
from sklearn.model_selection import KFold
import keras.backend as K
import optuna

from keras.utils import np_utils
from keras.models import Sequential, Model
from keras.layers import Input, Conv2D, MaxPooling2D, Dense, Dropout, Activation, Flatten
from keras.preprocessing.image import array_to_img, img_to_array, load_img
from keras.applications import ResNet50, VGG19
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras import optimizers
from sklearn.model_selection import train_test_split
from keras.backend import clear_session

import tensorflow as tf

batch_size = 8
nb_epochs = 1000
fold_num = 5

X = []
Y_pre = []
Y = np.zeros((1000,200))

with open('X_data.csv') as f:
    for row in csv.reader(f):
        X.append(row)

with open('Y_data.csv') as f:
    for row in csv.reader(f):
        Y_pre.append(row)

X = np.array(X)
Y_pre = np.array(Y_pre)



for i,num in enumerate(Y_pre):
    j = float(num[0]) / 0.005 - 1
    Y[i,int(j)]=1

print(X.shape)
print(Y.shape)


kf = KFold(n_splits=fold_num, shuffle=True)

def objective(trial):
    # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
    # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    # K.set_session(sess)

    #最適化するパラメータの設定

    #中間層１のユニット数
    hidden_neurons1 = int(trial.suggest_discrete_uniform("hidden_neurons1", 100, 5000, 100))

    hidden_neurons2 = int(trial.suggest_discrete_uniform("hidden_neurons2", 100, 5000, 100))

    # drop = trial.suggest_uniform("drop", 0.1, 0.5)

    out_neurons = 200

    #optimizer
    lr = trial.suggest_loguniform("lr", 1e-1, 1e-4)

    activation = trial.suggest_categorical("activation", ["linear", "sigmoid", "relu"])

    loss_all = []

    for train, test in kf.split(X):

        es = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')

        model = Sequential()
        model.add(Dense(hidden_neurons1, input_shape= (25088, )))
        model.add(Activation(activation))
        model.add(Dropout(0.5))
        model.add(Dense(hidden_neurons2))
        model.add(Activation(activation))
        model.add(Dropout(0.5))
        model.add(Dense(out_neurons))
        model.add(Activation("softmax"))
        model.compile(loss='categorical_crossentropy', metrics = ['accuracy'], optimizer=optimizers.Adam(lr=lr))

        history = model.fit(X[train],
                           Y[train],
                           batch_size=batch_size,
                           epochs=nb_epochs,
                           verbose=1,
                           validation_data=(X[test], Y[test]),
                           shuffle=True,
                           callbacks=[es])

        #検証用データに対する正答率が最大となるハイパーパラメータを求める
        loss = history.history["val_loss"]
        loss.sort()
        loss_all.append(loss[0])
        print(" ")
        print("Loss: "+str(loss[0]))
        print(" ")

        clear_session()
        del model, history
        gc.collect()


    print(loss_all)
    print("%.2f%% (+/- %.2f%%)" % (np.mean(loss_all), np.std(loss_all)))

    return np.mean(loss_all)

study = optuna.create_study()
study.optimize(objective, n_trials=50)
print("End!!")
print("All trials")
print(study.trials)
print("Best Parameters")
print(study.best_params)
print("Best Value")
print(study.best_value)

print("end of script")

入力に画像サイズの情報の付与

次に、入力を改良します。
入力（VGG19の出力）に画像のサイズを付与します。
具体的には、224/画像のheight or weightを入力値として２つ付与して、学習させます。

まずは、その情報を作成しました。
ソースコードをexp_trans.pyに示します。
また，学習プログラムをVGG19_train3.pyに示します。
学習及び評価結果は、次回の記事に記載します。

python:exp_trans.py
import numpy as np
import os
import re
import csv
import cv2

def list_pictures(directory, ext='jpg|jpeg|bmp|png|ppm'):
    return [os.path.join(root, f)
            for root, _, files in os.walk(directory) for f in files
            if re.match(r'([\w]+\.(?:' + ext + '))', f.lower())]

X = []

picture_hw = np.zeros((1000, 2))
path = './test_dataset/'
base_hw = 224
count = 0

path0 = './dataset/'
for picture in list_pictures(path):
    picture_name=float(picture[picture.find(path)+len(path):picture.find('_0_')])
    im = cv2.imread(path0+str(int(picture_name))+".jpg")
    h, w, c = im.shape
    picture_hw[count, 0] = base_hw/h
    picture_hw[count, 1] = base_hw/w

    count = count + 1

print(len(picture_hw))

X_post = []

with open('X_add_data.csv', 'w', newline="") as f:
    writer = csv.writer(f)
    writer.writerows(picture_hw)

with open('X_add_data.csv') as f:
    for row in csv.reader(f):
        X_post.append(row)

X_post = np.array(X_post)

print(X_post.shape)

python:VGG19_train3.py
import keras
import numpy as np
import matplotlib.pyplot as plt
import os
import re
import csv
import gc
from sklearn.model_selection import KFold
import keras.backend as K
import optuna

from keras.utils import np_utils
from keras.models import Sequential, Model
from keras.layers import Input, Conv2D, MaxPooling2D, Dense, Dropout, Activation, Flatten
from keras.preprocessing.image import array_to_img, img_to_array, load_img
from keras.applications import ResNet50, VGG19
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras import optimizers
from sklearn.model_selection import train_test_split
from keras.backend import clear_session

import tensorflow as tf

batch_size = 8
nb_epochs = 1000
fold_num = 5

X = []
X_add = []
Y_pre = []
Y = np.zeros((1000,200))

with open('X_data.csv') as f:
    for row in csv.reader(f):
        X.append(row)

with open('X_add_data.csv') as f:
    for row in csv.reader(f):
        X_add.append(row)

with open('Y_data.csv') as f:
    for row in csv.reader(f):
        Y_pre.append(row)

X = np.array(X)
X_add = np.array(X_add)
Y_pre = np.array(Y_pre)



for i,num in enumerate(Y_pre):
    j = float(num[0]) / 0.005 - 1
    Y[i,int(j)]=1


X = np.concatenate([X, X_add], 1)

print(X.shape)
print(Y.shape)

kf = KFold(n_splits=fold_num, shuffle=True)

def objective(trial):

    #最適化するパラメータの設定

    #中間層１のユニット数
    hidden_neurons1 = int(trial.suggest_discrete_uniform("hidden_neurons1", 100, 5000, 100))

    hidden_neurons2 = int(trial.suggest_discrete_uniform("hidden_neurons2", 100, 5000, 100))

    drop = trial.suggest_uniform("drop", 0.1, 0.5)

    out_neurons = 200

    #optimizer
    lr = trial.suggest_loguniform("lr", 1e-4, 1e-1)

    activation = trial.suggest_categorical("activation", ["linear", "sigmoid", "relu"])

    loss_all = []

    for train, test in kf.split(X):

        es = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')

        model = Sequential()
        model.add(Dense(hidden_neurons1, input_shape= (25090, )))
        model.add(Activation(activation))
        model.add(Dropout(drop))
        model.add(Dense(hidden_neurons2))
        model.add(Activation(activation))
        model.add(Dropout(drop))
        model.add(Dense(out_neurons))
        model.add(Activation("softmax"))
        model.compile(loss='categorical_crossentropy', metrics = ['accuracy'], optimizer=optimizers.Adam(lr=lr))

        history = model.fit(X[train],
                           Y[train],
                           batch_size=batch_size,
                           epochs=nb_epochs,
                           verbose=1,
                           validation_data=(X[test], Y[test]),
                           shuffle=True,
                           callbacks=[es])

        #検証用データに対する正答率が最大となるハイパーパラメータを求める
        loss = history.history["val_loss"]
        loss.sort()
        loss_all.append(loss[0])
        print(" ")
        print("Loss: "+str(loss[0]))
        print(" ")

        clear_session()
        del model, history
        gc.collect()


    print(loss_all)
    print("%.2f%% (+/- %.2f%%)" % (np.mean(loss_all), np.std(loss_all)))

    return np.mean(loss_all)

study = optuna.create_study()
study.optimize(objective, n_trials=50)
print("End!!")
print("All trials")
print(study.trials)
print("Best Parameters")
print(study.best_params)
print("Best Value")
print(study.best_value)

print("end of script")

アンサンブル学習の準備

アンサンブル学習の問題点としては、様々な機械学習アルゴリズムを動かさなくてはならないために、処理が重くなることが挙げられます。
そのため、アンサンブル学習を実装するよりも全結合層の検討等を行い、よりNNに対する検討を行うと同時に、それぞれの機械学習の精度を確認してアンサンブル学習を行う予定です。

機械学習アルゴリズムのラインナップはランダムフォレストとXGBTです。
これらはNNとは異なるアルゴリズムで、非線形的な表現も可能です。
これらを単体で試行するため、学習・検証プログラムをRF_train3.py、XGBT_train3.pyにそれぞれ示します。

python:RF_train3.py
import numpy as np
import matplotlib.pyplot as plt
import os
import re
import csv
import gc
from sklearn.model_selection import KFold
import optuna

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from pandas.plotting import scatter_matrix

X = []
X_add = []
Y_pre = []
Y = np.zeros((1000,200))

with open('X_data.csv') as f:
    for row in csv.reader(f):
        X.append(row)

with open('X_add_data.csv') as f:
    for row in csv.reader(f):
        X_add.append(row)

with open('Y_data.csv') as f:
    for row in csv.reader(f):
        Y_pre.append(row)

X = np.array(X)
X_add = np.array(X_add)
Y_pre = np.array(Y_pre)



for i,num in enumerate(Y_pre):
    j = float(num[0]) / 0.005 - 1
    Y[i,int(j)]=1


X = np.concatenate([X, X_add], 1)

print(X.shape)
print(Y.shape)

kf = KFold(n_splits=fold_num, shuffle=True)

def objective(trial):

    #最適化するパラメータの設定

    #中間層１のユニット数
    max_depth = int(trial.suggest_discrete_uniform("max_depth", 5, 500, 10))

    n_estimators = int(trial.suggest_discrete_uniform("n_estimators", 5, 500, 10))

    max_leaf_nodes = int(trial.suggest_discrete_uniform("max_leaf_nodes", 4, 64, 4))

    min_samples_split = trial.suggest_int("min_samples_split", 8, 16)

    criterion = trial.suggest_categorical("criterion", ["gini", "entropy"])

    acc_all = []

    for train, test in kf.split(X):

        random_forest = RandomForestClassifier(verbose=True,
                                                min_samples_split = min_samples_split,
                                                max_leaf_nodes = max_leaf_nodes,
                                                criterion = criterion,
                                                max_depth=max_depth,
                                                n_estimators=n_estimators,
                                                random_state=42)
        random_forest.fit(X[train], Y[train])

        # 予測値算出
        y_pred = random_forest.predict(X[test])

        #モデルを作成する段階でのモデルの識別精度
        trainaccuracy_random_forest = random_forest.score(X[train], Y[train])
        print('TrainAccuracy: {}'.format(trainaccuracy_random_forest))

        #作成したモデルに学習に使用していない評価用のデータセットを入力し精度を確認
        accuracy_random_forest = accuracy_score(Y[test], y_pred)
        print('Accuracy: {}'.format(accuracy_random_forest))

        acc_all.append(accuracy_random_forest)

        clear_session()
        del random_forest, y_pred, trainaccuracy_random_forest, accuracy_random_forest
        gc.collect()


    print(acc_all)

    return 1.0 - np.mean(loss_all)

study = optuna.create_study()
study.optimize(objective, n_trials=50)
print("End!!")
print("All trials")
print(study.trials)
print("Best Parameters")
print(study.best_params)
print("Best Value")
print(study.best_value)

print("end of script")

import numpy as np
import matplotlib.pyplot as plt
import xgboost as xgb
import os
import re
import csv
import gc
import optuna

from sklearn.model_selection import train_test_split, kFold
from sklearn.metrics import accuracy_score, precision_score
from sklearn.ensemble import RandomForestClassifier

X = []
X_add = []
Y_pre = []
Y = np.zeros((1000,200))

with open('X_data.csv') as f:
    for row in csv.reader(f):
        X.append(row)

with open('X_add_data.csv') as f:
    for row in csv.reader(f):
        X_add.append(row)

with open('Y_data.csv') as f:
    for row in csv.reader(f):
        Y_pre.append(row)

X = np.array(X)
X_add = np.array(X_add)
Y_pre = np.array(Y_pre)



for i,num in enumerate(Y_pre):
    j = float(num[0]) / 0.005 - 1
    Y[i,int(j)]=1


X = np.concatenate([X, X_add], 1)

print(X.shape)
print(Y.shape)

kf = KFold(n_splits=fold_num, shuffle=True)

def objective(trial):

    #最適化するパラメータの設定

    #中間層１のユニット数
    max_depth = trial.suggest_int('max_depth',1,30)
    learning_rate = trial.suggest_uniform('learning_rate',0.0,1)
    round_num = trial.suggest_int('round_num',1,30)

    acc_all = []

    for train, test in kf.split(X):

        train = xgb.DMatrix(X[train],label=Y[train])
        param = { 'max_depth':max_depth,'learning_rate':learning_rate,'objective':'reg:logistic' }
        bst = xgb.train(param,dtrain,num_round)

        ans_train = predict(bst,X[train]).round().astype(int)
        print('train score :',accuracy_score(ans_train.round(),Y[train]))
        ans_test = predict(bst,X[test]).round().astype(int)
        print('test score :',accuracy_score(ans_test.round(),Y[test]))

        acc_all.append(accuracy_score(ans_test.round(),Y[test]))

        clear_session()
        del train, param, bst, ans_test, ans_train
        gc.collect()


    print(acc_all)

    return 1 - np.mean(loss_all)

study = optuna.create_study()
study.optimize(objective, n_trials=50)
print("End!!")
print("All trials")
print(study.trials)
print("Best Parameters")
print(study.best_params)
print("Best Value")
print(study.best_value)

print("end of script")

次回の予定

• 次回は、アンサンブル学習を行います。

---

次の記事へ

前の記事へ 戻る