complex_siamese.py

from __future__ import absolute_import
from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

import random
from keras.datasets import mnist
from keras.models import Sequential, Graph
from keras.layers.core import Dense, Dropout, Lambda
from keras.optimizers import SGD, RMSprop
from keras import backend as K
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Flatten, Activation
from keras.layers.convolutional import Convolution3D, MaxPooling3D, ZeroPadding3D
from keras.optimizers import SGD

def weighted_average_euclidean_distance(inputs, weight_averages):
    branch_1 = None
    branch_2 = None
    for k,v in inputs:
        val = v
        val = val * weight_averages[k]
        if 'a' in k:
            if branch_1 is None:
                branch_1 = val
            else:
                branch_1 += val
        elif 'b' in k:
            if branch_2 is None:
                branch_2 =val
            else:
                branch_2 += val
    return K.sqrt(K.sum(K.square(branch_1 - branch_2), axis=1, keepdims=True))


def generate_column_weights():
    from mvpa2.misc.fx import double_gamma_hrf
    frames_step = 16.0
    FPS = 30.0
    hrf_fps_t = np.linspace(20,0, 20 *FPS/frames_step)
    ret = {}
    for i in range(len(hrf_fps_t)):
        ret[str(i) + 'a'] = double_gamma_hrf(hrf_fps_t[i])
        ret[str(i) + 'b'] = double_gamma_hrf(hrf_fps_t[i])
    return ret

def C3D_Sports1M():
    model = Sequential()
    # 1st layer group
    model.add(Convolution3D(64, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv1',
                            subsample=(1, 1, 1),
                            input_shape=(3, 16, 112, 112)))
    model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
                           border_mode='valid', name='pool1'))

    # 2nd layer group
    model.add(Convolution3D(128, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv2',
                            subsample=(1, 1, 1)))
    model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
                           border_mode='valid', name='pool2'))

    # 3rd layer group
    model.add(Convolution3D(256, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv3a',
                            subsample=(1, 1, 1)))
    model.add(Convolution3D(256, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv3b',
                            subsample=(1, 1, 1)))
    model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
                           border_mode='valid', name='pool3'))

    # 4th layer group
    model.add(Convolution3D(512, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv4a',
                            subsample=(1, 1, 1)))
    model.add(Convolution3D(512, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv4b',
                            subsample=(1, 1, 1)))
    model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
                           border_mode='valid', name='pool4'))

    # 5th layer group
    model.add(Convolution3D(512, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv5a',
                            subsample=(1, 1, 1)))
    model.add(Convolution3D(512, 3, 3, 3, activation='relu',
                            border_mode='same', name='conv5b',
                            subsample=(1, 1, 1)))
    model.add(ZeroPadding3D(padding=(0, 1, 1)))
    model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
                           border_mode='valid', name='pool5'))
    model.add(Flatten())

    # FC layers group
    model.add(Dense(4096, activation='relu', name='fc6'))
    model.add(Dropout(.5))
    model.add(Dense(4096, activation='relu', name='fc7'))
    model.add(Dropout(.5))
    model.add(Dense(487, activation='softmax', name='fc8'))

    return model

def contrastive_loss(y, d):
    '''Contrastive loss from Hadsell-et-al.'06
    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
    '''
    margin = 1
    return K.mean(y * K.square(d) + (1 - y) * K.square(K.maximum(margin - d, 0)))

def make_graph_network():
    input_dim=(3, 16, 112, 112)
    base_network = C3D_Sports1M()
    weights = generate_column_weights()
    input_names = weights.keys()
    output_names = []
    g = Graph()
    for i in input_names:
        output_names.append(i + '_output')
        g.add_input(name=i, input_shape=(input_dim,))
    g.add_shared_node(base_network, name='shared', inputs=input_names, outputs=output_names)
    g.add_node(Lambda(weighted_average_euclidean_distance, arguments={weight_averages: weights}), name='d', input='shared')
    g.add_output(name='output', input='d')

    rms = RMSprop()
    g.compile(loss={'output': contrastive_loss}, optimizer=rms)