GQN.py

#deep learning library
import keras
#3D Dataset
import deepmind_lab

#parameters
batch_size = 128
num_classes = 10
epochs = 12

# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = deepmind_lab.Lab('image_frames')

#Representation Network (Convolutional Network)
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
#optimize with gradient descent
model.compile(loss='categorical_crossentropy', optimizer='adam')

#Generation Network 
model2 = Sequential()
#The number of timestep sequence are dealt with in the fit function
model2.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
model2.add(Dropout(0.2))
#number of features on the output
model2.add(Dense(y.shape[1], activation='softmax'))
#optimize with gradient descent
model2.compile(loss='categorical_crossentropy', optimizer='adam')
model2.fit(X, y, epochs=5, batch_size=128)

#train the generation network on the outputs of the representation network (no labels)
model2.fit(model.fit(x_train)) 

#Now we give the generatior a new image, and it will predict ("imagine") a 3D map!
model2.predict('/my_input_image.jpg')