noinputerror.java

package com.tactico.tm.nn.graphs;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;

import org.deeplearning4j.api.storage.StatsStorage;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.BackpropType;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.GradientNormalization;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.graph.MergeVertex;
import org.deeplearning4j.nn.conf.graph.rnn.DuplicateToTimeSeriesVertex;
import org.deeplearning4j.nn.conf.graph.rnn.LastTimeStepVertex;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.ActivationLayer;
import org.deeplearning4j.nn.conf.layers.ConvolutionLayer;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer;
import org.deeplearning4j.nn.conf.layers.GravesLSTM;
import org.deeplearning4j.nn.conf.layers.LocalResponseNormalization;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.conf.layers.PoolingType;
import org.deeplearning4j.nn.conf.layers.RnnOutputLayer;
import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.deeplearning4j.ui.stats.StatsListener;
import org.deeplearning4j.ui.storage.InMemoryStatsStorage;
import org.deeplearning4j.util.ModelSerializer;
import org.json.simple.parser.ParseException;
import org.deeplearning4j.ui.api.UIServer;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.MultiDataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;

import com.tactico.backtest.BacktestFactory;
import com.tactico.tm.nn.Constants;


public class RecurrentRegressionComputationGraph {
    public Configuration conf;
    
	public ComputationGraph buildCG (int numInputs, int numClasses, double learningRate){
		conf = new Configuration(numInputs);
		conf.learningRate = learningRate;
		
		ComputationGraphConfiguration.GraphBuilder confBuilder = new NeuralNetConfiguration.Builder()
                .iterations(1)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .learningRate(conf.learningRate)
                .updater(Updater.ADAM).adamMeanDecay(1. - 1./10.).adamVarDecay(1. - 1./50.)
                .weightInit(WeightInit.XAVIER_UNIFORM)
                //.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                .regularization(true)
                .l2(conf.l2).l1(conf.l1).dropOut(0.9)
                .graphBuilder()
                .setInputTypes(InputType.recurrent(numInputs))
                .addInputs("input");
		String[][] lstmStacks = new String[conf.numStacks][conf.depthStack];
		String[][] poolStacks = new String[conf.numStacks][conf.depthStack-1];
		for (int stackIdx = 0; stackIdx < conf.numStacks; stackIdx++){
			for (int i = 0; i < conf.depthStack; i++) {
				lstmStacks[stackIdx][i] = "lstm-"+stackIdx+"-"+i;
				//
				// Input is input + previous pooling 
				GravesLSTM lstmLayer = new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
	            		.nIn(i==0 ? numInputs : conf.numHiddenNodes[i-1])
		    			.nOut(conf.numHiddenNodes[i]).forgetGateBiasInit(5.)
		    			.activation(Activation.TANH)
		    			.build();
				if (i == 0){
					confBuilder.addLayer(lstmStacks[stackIdx][i], lstmLayer, "input");
				} else {
					confBuilder.addLayer(lstmStacks[stackIdx][i], lstmLayer, lstmStacks[stackIdx][i-1]);
				}
				if (i < conf.depthStack-1){
		            poolStacks[stackIdx][i] = "pool-"+stackIdx+"-"+i;
		            //confBuilder.addLayer(poolStacks[stackIdx][i], new GlobalPoolingLayer.Builder(PoolingType.MAX).collapseDimensions(true).build(), lstmStacks[stackIdx][i]);
		            //confBuilder.addVertex(poolStacks[stackIdx][i], new DuplicateToTimeSeriesVertex("input"), "pool1-"+stackIdx+"-"+i);
				}
	        }
		}
		String[] stackEnds = new String[conf.numStacks*2];
		for (int stackIdx = 0; stackIdx < conf.numStacks; stackIdx++){
			stackEnds[stackIdx] = lstmStacks[stackIdx][conf.depthStack-1];
		}
		for (int stackIdx = 0; stackIdx < conf.numStacks; stackIdx++){
			stackEnds[stackIdx+conf.numStacks] = lstmStacks[stackIdx][0];
		}
		confBuilder.addLayer("output", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
			        .nIn(conf.numHiddenNodes[conf.depthStack-1]*conf.numStacks+conf.numHiddenNodes[0]*conf.numStacks)
			        .nOut(numClasses)
			        .activation(Activation.IDENTITY)
			        .build(), stackEnds);
		confBuilder.setOutputs("output");
		
		ComputationGraphConfiguration cgconf = confBuilder.pretrain(false)
				.backprop(true).backpropType(BackpropType.Standard)
				//.tBPTTForwardLength(500).tBPTTBackwardLength(500)
				.build();
		cgconf.addPreProcessors(InputType.recurrent(numInputs));
        ComputationGraph model = new ComputationGraph(cgconf);
        model.init();
		
        StatsStorage statsStorage = new InMemoryStatsStorage(); 
		//StatsStorage statsStorage = new FileStatsStorage(File("statsBug","w"))
        //Alternative: new FileStatsStorage(File) - see UIStorageExample
        int listenerFrequency = 1;
        UIServer uiServer = UIServer.getInstance();
        StatsListener statsList = new StatsListener(statsStorage, listenerFrequency);
        System.out.println("Session ID: "+statsList.getSessionID());
        model.setListeners(statsList,
        		new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
        //Attach the StatsStorage instance to the UI: this allows the contents of the StatsStorage to be visualized
        uiServer.attach(statsStorage);
        //model.setListeners(new ScoreIterationListener(Constants.NEURAL_NET_ITERATION_LISTENER));
        //model.setListeners(new StatsListener(statsStorage));
        System.out.println("Machine number of params:"+(model.numParams()));
        System.out.println("Learning rate: "+learningRate);
        return model;  
	}
	
    class Configuration {
    	int numStacks = 1;
    	int depthStack;
    	int[] numHiddenNodes;
        
        public double learningRate=1e-2;
        double l2 = 0.001;//minimal weight gaussian prior
        double l1 = 0.0001;//sparse weights laplace prior
        public Configuration(int numInput){
        	this.numHiddenNodes = new int[] {15};//defined layers
        	this.depthStack = this.numHiddenNodes.length;
        }

    }
    
    
    
    public static void main (String[] args){
		ComputationGraph cg = new RecurrentRegressionComputationGraph().buildCG(9,3,1e-2);
		String modelFile = String.format("IntermediateModels/bbrp_rc_KL_%1.3e",1e-2);
				
		MultiDataSet dataSet = new MultiDataSet();
		for (int pass = 0; pass < 10; pass++){
			System.out.println("Starting pass: "+(pass+1));
			for (int i = 1; i <= 10; i++){
				try {
					dataSet.load(new File("trainingSets/bbrp_"+i));
				} catch (IOException e1) {
					e1.printStackTrace();
				}
				INDArray[] dataSetInputs = dataSet.getFeatures();
				cg.setInputs(dataSetInputs);
				double score0 = cg.score(dataSet,true);
				double score = score0;
				int iter = 0;
				while (iter++ < 50 && score > 0.5*score0){
					cg.fit(dataSet);
					score = cg.score();
				}
				try {
					System.out.println("Saving model to: "+modelFile);
		            ModelSerializer.writeModel(cg, modelFile, true);
		        } catch (IOException e) {
		            e.printStackTrace();
		        }

				
			}
		}

		
	}

}