Last active
May 13, 2017 11:04
-
-
Save Joshuaalbert/b36ab4ec2de3dd27a59ee7ea91f0f394 to your computer and use it in GitHub Desktop.
with this conf the ui doesn't update
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package com.tactico.tm.nn.graphs.confs; | |
| import org.deeplearning4j.api.storage.StatsStorage; | |
| import org.deeplearning4j.nn.api.OptimizationAlgorithm; | |
| import org.deeplearning4j.nn.conf.BackpropType; | |
| import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; | |
| import org.deeplearning4j.nn.conf.NeuralNetConfiguration; | |
| import org.deeplearning4j.nn.conf.Updater; | |
| import org.deeplearning4j.nn.conf.inputs.InputType; | |
| import org.deeplearning4j.nn.conf.layers.GravesLSTM; | |
| import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; | |
| import org.deeplearning4j.nn.graph.ComputationGraph; | |
| import org.deeplearning4j.nn.transferlearning.TransferLearning; | |
| import org.deeplearning4j.nn.weights.WeightInit; | |
| import org.deeplearning4j.optimize.listeners.ScoreIterationListener; | |
| import org.deeplearning4j.ui.api.UIServer; | |
| import org.deeplearning4j.ui.stats.StatsListener; | |
| import org.deeplearning4j.ui.storage.InMemoryStatsStorage; | |
| import org.nd4j.linalg.activations.Activation; | |
| import org.nd4j.linalg.lossfunctions.LossFunctions; | |
| public class ValueRangeConf { | |
| public double learningRate=1e-2; | |
| double l2 = 0.001;//minimal weight gaussian prior | |
| double l1 = 0.0001;//sparse weights laplace prior | |
| public int numInputs, numOutputs; | |
| boolean doDropout = false; | |
| double dropout = 1.0; | |
| public boolean isDoDropout() { | |
| return doDropout; | |
| } | |
| public void setDoDropout(boolean doDropout) { | |
| this.doDropout = doDropout; | |
| } | |
| public double getDropout() { | |
| return dropout; | |
| } | |
| public void setDropout(double dropout) { | |
| this.dropout = dropout; | |
| } | |
| public double getL2() { | |
| return l2; | |
| } | |
| public void setL2(double l2) { | |
| this.l2 = l2; | |
| } | |
| public double getL1() { | |
| return l1; | |
| } | |
| public void setL1(double l1) { | |
| this.l1 = l1; | |
| } | |
| public int getNumInputs() { | |
| return numInputs; | |
| } | |
| public void setNumInputs(int numInputs) { | |
| this.numInputs = numInputs; | |
| } | |
| public int getNumOutputs() { | |
| return numOutputs; | |
| } | |
| public void setNumOutputs(int numOutputs) { | |
| this.numOutputs = numOutputs; | |
| } | |
| public double getLearningRate(){ | |
| return learningRate; | |
| } | |
| public void setLearningRate(double learningRate) { | |
| this.learningRate = learningRate; | |
| } | |
| int numStateInput,numWindows; | |
| int depthMain,depthDiff,depthMean,depthMax,depthMin,depthControl; | |
| boolean doMainSkip, doDiffSkip, doMeanSkip, doMinSkip, doMaxSkip, doControlSkip; | |
| public boolean isDoDiffSkip() { | |
| return doDiffSkip; | |
| } | |
| public void setDoDiffSkip(boolean doDiffSkip) { | |
| this.doDiffSkip = doDiffSkip; | |
| } | |
| public boolean isDoMeanSkip() { | |
| return doMeanSkip; | |
| } | |
| public void setDoMeanSkip(boolean doMeanSkip) { | |
| this.doMeanSkip = doMeanSkip; | |
| } | |
| public boolean isDoMinSkip() { | |
| return doMinSkip; | |
| } | |
| public void setDoMinSkip(boolean doMinSkip) { | |
| this.doMinSkip = doMinSkip; | |
| } | |
| public boolean isDoMaxSkip() { | |
| return doMaxSkip; | |
| } | |
| public void setDoMaxSkip(boolean doMaxSkip) { | |
| this.doMaxSkip = doMaxSkip; | |
| } | |
| public boolean isDoControlSkip() { | |
| return doControlSkip; | |
| } | |
| public void setDoControlSkip(boolean doControlSkip) { | |
| this.doControlSkip = doControlSkip; | |
| } | |
| public boolean isDoMainSkip() { | |
| return doMainSkip; | |
| } | |
| public void setDoMainSkip(boolean doMainSkip) { | |
| this.doMainSkip = doMainSkip; | |
| } | |
| int[] mainStackNodes, diffStackNodes, meanStackNodes,maxStackNodes,minStackNodes,controlStackNodes; | |
| public int[] getControlStackNodes() { | |
| return controlStackNodes; | |
| } | |
| public void setControlStackNodes(int[] controlStackNodes) { | |
| this.controlStackNodes = controlStackNodes; | |
| this.depthControl = controlStackNodes.length; | |
| } | |
| public int getDepthControl() { | |
| return depthControl; | |
| } | |
| public int[] getDiffStackNodes() { | |
| return diffStackNodes; | |
| } | |
| public void setDiffStackNodes(int[] diffStackNodes) { | |
| this.diffStackNodes = new int[diffStackNodes.length+1]; | |
| for (int i=0; i < diffStackNodes.length; i++){ | |
| this.diffStackNodes[i] = diffStackNodes[i]; | |
| } | |
| this.diffStackNodes[diffStackNodes.length] = getNumInputs()*getNumWindows(); | |
| this.depthDiff = this.diffStackNodes.length; | |
| } | |
| public int getNumWindows() { | |
| return numWindows; | |
| } | |
| public void setNumWindows(int numWindows) { | |
| this.numWindows = numWindows; | |
| } | |
| public int[] getMeanStackNodes() { | |
| return meanStackNodes; | |
| } | |
| public void setMeanStackNodes(int[] meanStackNodes) { | |
| this.meanStackNodes = new int[meanStackNodes.length+1]; | |
| for (int i=0; i < meanStackNodes.length; i++){ | |
| this.meanStackNodes[i] = meanStackNodes[i]; | |
| } | |
| this.meanStackNodes[meanStackNodes.length] = getNumInputs()*getNumWindows(); | |
| this.depthMean = this.meanStackNodes.length; | |
| } | |
| public int[] getMaxStackNodes() { | |
| return maxStackNodes; | |
| } | |
| public void setMaxStackNodes(int[] maxStackNodes) { | |
| this.maxStackNodes = new int[maxStackNodes.length+1]; | |
| for (int i=0; i < maxStackNodes.length; i++){ | |
| this.maxStackNodes[i] = maxStackNodes[i]; | |
| } | |
| this.maxStackNodes[maxStackNodes.length] = getNumInputs()*getNumWindows(); | |
| this.depthMax = this.maxStackNodes.length; | |
| } | |
| public int[] getMinStackNodes() { | |
| return minStackNodes; | |
| } | |
| public void setMinStackNodes(int[] minStackNodes) { | |
| this.minStackNodes = new int[minStackNodes.length+1]; | |
| for (int i=0; i < minStackNodes.length; i++){ | |
| this.minStackNodes[i] = minStackNodes[i]; | |
| } | |
| this.minStackNodes[minStackNodes.length] = getNumInputs()*getNumWindows(); | |
| this.depthMin = this.minStackNodes.length; | |
| } | |
| public int getDepthDiff() { | |
| return depthDiff; | |
| } | |
| public int getDepthMean() { | |
| return depthMean; | |
| } | |
| public int getDepthMax() { | |
| return depthMax; | |
| } | |
| public int getDepthMin() { | |
| return depthMin; | |
| } | |
| public int getDepthMain() { | |
| return depthMain; | |
| } | |
| public int[] getMainStackNodes() { | |
| return mainStackNodes; | |
| } | |
| public void setMainStackNodes(int[] mainStackNodes) { | |
| this.mainStackNodes = mainStackNodes; | |
| this.depthMain = mainStackNodes.length; | |
| } | |
| public int getNumStateInput() { | |
| return numStateInput; | |
| } | |
| public void setNumStateInput(int numStateInput) { | |
| this.numStateInput = numStateInput; | |
| } | |
| public ValueRangeConf(int numInputs, int numStates, int numWindows){ | |
| this.numInputs = numInputs; | |
| this.numOutputs = numOutputs; | |
| this.setNumWindows(numWindows); | |
| this.setNumStateInput(numStates); | |
| //defaults | |
| this.setMainStackNodes(new int[] {5,5,4}); | |
| this.setDiffStackNodes(new int[] {3,4}); | |
| this.setMeanStackNodes(new int[] {3,4}); | |
| this.setMinStackNodes(new int[] {3,4}); | |
| this.setMaxStackNodes(new int[] {3,4}); | |
| this.setControlStackNodes(new int[] {2,2,2}); | |
| this.setDoControlSkip(false); | |
| this.setDoMeanSkip(false); | |
| this.setDoMinSkip(false); | |
| this.setDoMaxSkip(false); | |
| this.setDoDiffSkip(false); | |
| this.setDoMainSkip(false); | |
| } | |
| public ComputationGraphConfiguration getConf(){ | |
| ComputationGraphConfiguration.GraphBuilder confBuilder = new NeuralNetConfiguration.Builder() | |
| .iterations(1) | |
| .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) | |
| .learningRate(getLearningRate()) | |
| .updater(Updater.ADAM).adamMeanDecay(1. - 1./10.).adamVarDecay(1. - 1./50.) | |
| .weightInit(WeightInit.XAVIER_UNIFORM) | |
| //.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) | |
| .regularization(true).useDropConnect(true) | |
| .l2(getL2()).l1(getL1()).dropOut(getDropout()) | |
| .graphBuilder() | |
| .setInputTypes(InputType.recurrent(getNumInputs()),InputType.recurrent(getNumStateInput())) | |
| .addInputs("X","State"); | |
| String[] DiffStack = new String[getDepthDiff()]; | |
| String[] MeanStack = new String[getDepthMean()]; | |
| String[] MaxStack = new String[getDepthMax()]; | |
| String[] MinStack = new String[getDepthMin()]; | |
| String[] MainStack = new String[getDepthMain()]; | |
| String[] ControlStack = new String[getDepthControl()]; | |
| ControlStack[0] = "ControlStack-"+0; | |
| confBuilder.addLayer(ControlStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getNumInputs()+getNumStateInput()) | |
| .nOut(getControlStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),"X","State"); | |
| for(int i = 1; i < getDepthControl(); i++){ | |
| ControlStack[i] = "ControlStack-"+i; | |
| if (isDoControlSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getControlStackNodes()[i-1] + getControlStackNodes()[0]) | |
| .nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), ControlStack[0], ControlStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getControlStackNodes()[0]) | |
| .nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), ControlStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getControlStackNodes()[i-1]) | |
| .nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), ControlStack[i-1]); | |
| } | |
| } | |
| MinStack[0] = "MinStack-"+0; | |
| confBuilder.addLayer(MinStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1]) | |
| .nOut(getMinStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),"X",ControlStack[getDepthControl()-1]); | |
| for(int i = 1; i < getDepthMin(); i++){ | |
| MinStack[i] = "MinStack-"+i; | |
| if (isDoMinSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMinStackNodes()[i-1] + getMinStackNodes()[0]) | |
| .nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MinStack[0], MinStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMinStackNodes()[0]) | |
| .nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MinStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMinStackNodes()[i-1]) | |
| .nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MinStack[i-1]); | |
| } | |
| } | |
| MaxStack[0] = "MaxStack-"+0; | |
| confBuilder.addLayer(MaxStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1]) | |
| .nOut(getMaxStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),"X",ControlStack[getDepthControl()-1]); | |
| for(int i = 1; i < getDepthMax(); i++){ | |
| MaxStack[i] = "MaxStack-"+i; | |
| if (isDoMaxSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMaxStackNodes()[i-1] + getMaxStackNodes()[0]) | |
| .nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MaxStack[0], MaxStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMaxStackNodes()[0]) | |
| .nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MaxStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMaxStackNodes()[i-1]) | |
| .nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MaxStack[i-1]); | |
| } | |
| } | |
| MeanStack[0] = "MeanStack-"+0; | |
| confBuilder.addLayer(MeanStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1]) | |
| .nOut(getMeanStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),"X",ControlStack[getDepthControl()-1]); | |
| for(int i = 1; i < getDepthMean(); i++){ | |
| MeanStack[i] = "MeanStack-"+i; | |
| if (isDoMeanSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMeanStackNodes()[i-1] + getMeanStackNodes()[0]) | |
| .nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MeanStack[0], MeanStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMeanStackNodes()[0]) | |
| .nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MeanStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMeanStackNodes()[i-1]) | |
| .nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MeanStack[i-1]); | |
| } | |
| } | |
| DiffStack[0] = "DiffStack-"+0; | |
| confBuilder.addLayer(DiffStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMinStackNodes()[getDepthMin()-1] + getMaxStackNodes()[getDepthMax()-1] + getControlStackNodes()[getDepthControl()-1]) | |
| .nOut(getDiffStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),MinStack[getDepthMin()-1], MaxStack[getDepthMax()-1],ControlStack[getDepthControl()-1]); | |
| for(int i = 1; i < getDepthDiff(); i++){ | |
| DiffStack[i] = "DiffStack-"+i; | |
| if (isDoDiffSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getDiffStackNodes()[i-1] + getDiffStackNodes()[0]) | |
| .nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), DiffStack[0], DiffStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getDiffStackNodes()[0]) | |
| .nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), DiffStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getDiffStackNodes()[i-1]) | |
| .nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), DiffStack[i-1]); | |
| } | |
| } | |
| MainStack[0] = "MainStack-"+0; | |
| confBuilder.addLayer(MainStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getNumInputs() + getDiffStackNodes()[getDepthDiff()-1] + getMeanStackNodes()[getDepthMean()-1] + getControlStackNodes()[getDepthControl()-1]) | |
| .nOut(getMainStackNodes()[0]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(),"X", DiffStack[getDepthDiff()-1], MeanStack[getDepthMean()-1], ControlStack[getDepthControl()-1]); | |
| for(int i = 1; i < getDepthMain(); i++){ | |
| MainStack[i] = "MainStack-"+i; | |
| if (isDoMainSkip()){ | |
| if (i > 1){ | |
| confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMainStackNodes()[i-1] + getMainStackNodes()[0]) | |
| .nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MainStack[0], MainStack[i-1]); | |
| } else { | |
| confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMainStackNodes()[0]) | |
| .nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MainStack[0]); | |
| } | |
| } else { | |
| confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID) | |
| .nIn(getMainStackNodes()[i-1]) | |
| .nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.) | |
| .activation(Activation.TANH) | |
| .build(), MainStack[i-1]); | |
| } | |
| } | |
| // outputs | |
| confBuilder.addLayer("out1", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE) | |
| .nIn(getMainStackNodes()[getDepthMain()-1]) | |
| .nOut(1) | |
| .activation(Activation.IDENTITY) | |
| .build(), MainStack[getDepthMain()-1]); | |
| confBuilder.addLayer("out2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE) | |
| .nIn(getMainStackNodes()[getDepthMain()-1]) | |
| .nOut(1) | |
| .activation(Activation.IDENTITY) | |
| .build(), MainStack[getDepthMain()-1]); | |
| confBuilder.addLayer("out3", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE) | |
| .nIn(getMainStackNodes()[getDepthMain()-1]) | |
| .nOut(1) | |
| .activation(Activation.IDENTITY) | |
| .build(), MainStack[getDepthMain()-1]); | |
| confBuilder.addLayer("out4", new RnnOutputLayer.Builder(LossFunctions.LossFunction.KL_DIVERGENCE) | |
| .nIn(getMainStackNodes()[getDepthMain()-1]) | |
| .nOut(1) | |
| .activation(Activation.SIGMOID) | |
| .build(), MainStack[getDepthMain()-1]); | |
| confBuilder.setOutputs("out1","out2","out3","out4"); | |
| ComputationGraphConfiguration cgconf = confBuilder.pretrain(false) | |
| .backprop(true).backpropType(BackpropType.Standard) | |
| //.tBPTTForwardLength(500).tBPTTBackwardLength(500) | |
| .build(); | |
| return cgconf; | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment