Keras ConvLSTM2D network¶
Use a network made of convolutional LSTM layers.
In [13]:
import numpy as np
import tensorflow as tf
from keras.models import *
from keras.layers import *
from keras.optimizers import *
from keras.utils.np_utils import to_categorical
from keras.utils import plot_model
import keras.backend as K
import imageio
from PIL import Image
from matplotlib.pyplot import imshow
%matplotlib inline
import random
Model¶
In [2]:
#kernel_size = (3,3)
#img_input = Input(shape=(None,None,None,1,))
#x = ConvLSTM2D(32, kernel_size, activation='relu', padding='same', return_sequences=True)(img_input)
#x = BatchNormalization()(x)
#x = ConvLSTM2D(32, kernel_size, activation='relu', padding='same', return_sequences=True)(x)
#x = BatchNormalization()(x)
#x = ConvLSTM2D(32, kernel_size, activation='relu', padding='same', return_sequences=True)(x)
#x = BatchNormalization()(x)
#x = ConvLSTM2D(32, kernel_size, activation='relu', padding='same', return_sequences=True)(x)
#x = BatchNormalization()(x)
#x = ConvLSTM2D(1, kernel_size, activation='softmax', padding='same', return_sequences=False)(x)
#model = Model(inputs=img_input, outputs=x)
#model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy')
#model.summary()
seq = Sequential()
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
input_shape=(None, None, None, 1),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(Conv3D(filters=1, kernel_size=(3, 3, 3),
activation='sigmoid',
padding='same', data_format='channels_last'))
seq.compile(loss='binary_crossentropy', optimizer='adadelta')
model = seq
model.summary()
In [3]:
for layer in model.layers:
print(layer.output_shape)
In [5]:
model = load_model("model/kerasExampleDots.h5")
Data¶
Full games are represented as image sequences ("movies"). The network has to predict the next frame of an unfinished sequence.
The input data is the full game without the last state where all lines are filled in. The output data is the full game without the very first state where no lines are drawn.
In [6]:
sequenceDataset = np.load('sequence5x4.npz')
games = sequenceDataset['games']
print(games.shape)
In [7]:
x_train = games[:,:-1,:,:]
y_train = games[:,1:,:,:]
print(x_train.shape)
print(y_train.shape)
x_train = x_train.astype(K.floatx())
y_train = y_train.astype(K.floatx())
x_train /= 255
y_train /= 255
In [8]:
np.set_printoptions(precision=2, suppress=True, linewidth=90)
exampleGameIdx = 23
exampleGameFrame = 42
print(x_train[exampleGameIdx,exampleGameFrame])
print(y_train[exampleGameIdx,exampleGameFrame])
In [9]:
channel_shape = x_train.shape + (1,)
x_train = x_train.reshape(channel_shape)
#cat_shape = y_train.shape + (2,)
#y_train = to_categorical(y_train).reshape(cat_shape)
cat_shape = y_train.shape + (1,)
y_train = y_train.reshape(cat_shape)
print(x_train.shape)
print(y_train.shape)
print(np.transpose(x_train[exampleGameIdx,exampleGameFrame,:,:,0]))
print(np.transpose(y_train[exampleGameIdx,exampleGameFrame,:,:,0]))
#print(np.transpose(y_train[exampleGameIdx,exampleGameFrame,:,:,1]))
Training¶
In [8]:
#model.fit(x_train[0:8:1,:,:,:,:], y_train[0:8:1,:,:,:,:], epochs=50, batch_size=16)
model.fit(x_train[:900], y_train[:900], batch_size=10,
epochs=200, validation_split=0.05)
Out[8]:
In [23]:
example = random.randrange(x_train.shape[0])
exampleFrame = 20
input_data = np.array([x_train[example,0:exampleFrame,::,::,::]])
prediction = model.predict(input_data)
print(prediction.shape)
prediction = prediction[0,-1, ::, ::, 0]
print(prediction.shape)
print(prediction)
print(x_train[example,exampleFrame,::,::,0])
# create image
target_imgdata = x_train[example,exampleFrame,::,::,0] * 255
target_imgdata = target_imgdata.astype(np.uint8)
prediction_imgdata = prediction * 255
prediction_imgdata = prediction_imgdata.astype(np.uint8)
# merge image data in color channels
tmp = np.zeros(prediction.shape, dtype=np.uint8)
merged_imgdata = np.stack([target_imgdata, prediction_imgdata, tmp], axis=2)
merged_imgdata_large = np.append(target_imgdata, prediction_imgdata, axis=1)
print(merged_imgdata_large.shape)
#create image
img = Image.fromarray(merged_imgdata, 'RGB')
img = Image.fromarray(merged_imgdata_large, 'P')
img = img.resize(size=(img.size[0]*10, img.size[1]*10))
img
Out[23]:
In [10]:
#model.save('model/kerasExampleDots2.h5')
In [ ]:
target_imgdata = x_train[example,:,:,0] * 255
target_imgdata = target_imgdata.astype(np.uint8)
prediction_imgdata = prediction[0] * 255
prediction_imgdata = prediction_imgdata.astype(np.uint8)
# merge image data in color channels
tmp = np.zeros((prediction[0].shape[0], prediction[0].shape[1]), dtype=np.uint8)
merged_imgdata = np.stack([target_imgdata, prediction_imgdata[:,:,1], tmp], axis=2)
#create image
img = Image.fromarray(merged_imgdata, 'RGB')
img = img.resize(size=(img.size[0]*10, img.size[1]*10))
img