Skip to content

Instantly share code, notes, and snippets.

@Joshuaalbert
Last active May 13, 2017 11:04
Show Gist options
  • Select an option

  • Save Joshuaalbert/b36ab4ec2de3dd27a59ee7ea91f0f394 to your computer and use it in GitHub Desktop.

Select an option

Save Joshuaalbert/b36ab4ec2de3dd27a59ee7ea91f0f394 to your computer and use it in GitHub Desktop.
with this conf the ui doesn't update
package com.tactico.tm.nn.graphs.confs;
import org.deeplearning4j.api.storage.StatsStorage;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.BackpropType;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.GravesLSTM;
import org.deeplearning4j.nn.conf.layers.RnnOutputLayer;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.transferlearning.TransferLearning;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.deeplearning4j.ui.api.UIServer;
import org.deeplearning4j.ui.stats.StatsListener;
import org.deeplearning4j.ui.storage.InMemoryStatsStorage;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.lossfunctions.LossFunctions;
public class ValueRangeConf {
public double learningRate=1e-2;
double l2 = 0.001;//minimal weight gaussian prior
double l1 = 0.0001;//sparse weights laplace prior
public int numInputs, numOutputs;
boolean doDropout = false;
double dropout = 1.0;
public boolean isDoDropout() {
return doDropout;
}
public void setDoDropout(boolean doDropout) {
this.doDropout = doDropout;
}
public double getDropout() {
return dropout;
}
public void setDropout(double dropout) {
this.dropout = dropout;
}
public double getL2() {
return l2;
}
public void setL2(double l2) {
this.l2 = l2;
}
public double getL1() {
return l1;
}
public void setL1(double l1) {
this.l1 = l1;
}
public int getNumInputs() {
return numInputs;
}
public void setNumInputs(int numInputs) {
this.numInputs = numInputs;
}
public int getNumOutputs() {
return numOutputs;
}
public void setNumOutputs(int numOutputs) {
this.numOutputs = numOutputs;
}
public double getLearningRate(){
return learningRate;
}
public void setLearningRate(double learningRate) {
this.learningRate = learningRate;
}
int numStateInput,numWindows;
int depthMain,depthDiff,depthMean,depthMax,depthMin,depthControl;
boolean doMainSkip, doDiffSkip, doMeanSkip, doMinSkip, doMaxSkip, doControlSkip;
public boolean isDoDiffSkip() {
return doDiffSkip;
}
public void setDoDiffSkip(boolean doDiffSkip) {
this.doDiffSkip = doDiffSkip;
}
public boolean isDoMeanSkip() {
return doMeanSkip;
}
public void setDoMeanSkip(boolean doMeanSkip) {
this.doMeanSkip = doMeanSkip;
}
public boolean isDoMinSkip() {
return doMinSkip;
}
public void setDoMinSkip(boolean doMinSkip) {
this.doMinSkip = doMinSkip;
}
public boolean isDoMaxSkip() {
return doMaxSkip;
}
public void setDoMaxSkip(boolean doMaxSkip) {
this.doMaxSkip = doMaxSkip;
}
public boolean isDoControlSkip() {
return doControlSkip;
}
public void setDoControlSkip(boolean doControlSkip) {
this.doControlSkip = doControlSkip;
}
public boolean isDoMainSkip() {
return doMainSkip;
}
public void setDoMainSkip(boolean doMainSkip) {
this.doMainSkip = doMainSkip;
}
int[] mainStackNodes, diffStackNodes, meanStackNodes,maxStackNodes,minStackNodes,controlStackNodes;
public int[] getControlStackNodes() {
return controlStackNodes;
}
public void setControlStackNodes(int[] controlStackNodes) {
this.controlStackNodes = controlStackNodes;
this.depthControl = controlStackNodes.length;
}
public int getDepthControl() {
return depthControl;
}
public int[] getDiffStackNodes() {
return diffStackNodes;
}
public void setDiffStackNodes(int[] diffStackNodes) {
this.diffStackNodes = new int[diffStackNodes.length+1];
for (int i=0; i < diffStackNodes.length; i++){
this.diffStackNodes[i] = diffStackNodes[i];
}
this.diffStackNodes[diffStackNodes.length] = getNumInputs()*getNumWindows();
this.depthDiff = this.diffStackNodes.length;
}
public int getNumWindows() {
return numWindows;
}
public void setNumWindows(int numWindows) {
this.numWindows = numWindows;
}
public int[] getMeanStackNodes() {
return meanStackNodes;
}
public void setMeanStackNodes(int[] meanStackNodes) {
this.meanStackNodes = new int[meanStackNodes.length+1];
for (int i=0; i < meanStackNodes.length; i++){
this.meanStackNodes[i] = meanStackNodes[i];
}
this.meanStackNodes[meanStackNodes.length] = getNumInputs()*getNumWindows();
this.depthMean = this.meanStackNodes.length;
}
public int[] getMaxStackNodes() {
return maxStackNodes;
}
public void setMaxStackNodes(int[] maxStackNodes) {
this.maxStackNodes = new int[maxStackNodes.length+1];
for (int i=0; i < maxStackNodes.length; i++){
this.maxStackNodes[i] = maxStackNodes[i];
}
this.maxStackNodes[maxStackNodes.length] = getNumInputs()*getNumWindows();
this.depthMax = this.maxStackNodes.length;
}
public int[] getMinStackNodes() {
return minStackNodes;
}
public void setMinStackNodes(int[] minStackNodes) {
this.minStackNodes = new int[minStackNodes.length+1];
for (int i=0; i < minStackNodes.length; i++){
this.minStackNodes[i] = minStackNodes[i];
}
this.minStackNodes[minStackNodes.length] = getNumInputs()*getNumWindows();
this.depthMin = this.minStackNodes.length;
}
public int getDepthDiff() {
return depthDiff;
}
public int getDepthMean() {
return depthMean;
}
public int getDepthMax() {
return depthMax;
}
public int getDepthMin() {
return depthMin;
}
public int getDepthMain() {
return depthMain;
}
public int[] getMainStackNodes() {
return mainStackNodes;
}
public void setMainStackNodes(int[] mainStackNodes) {
this.mainStackNodes = mainStackNodes;
this.depthMain = mainStackNodes.length;
}
public int getNumStateInput() {
return numStateInput;
}
public void setNumStateInput(int numStateInput) {
this.numStateInput = numStateInput;
}
public ValueRangeConf(int numInputs, int numStates, int numWindows){
this.numInputs = numInputs;
this.numOutputs = numOutputs;
this.setNumWindows(numWindows);
this.setNumStateInput(numStates);
//defaults
this.setMainStackNodes(new int[] {5,5,4});
this.setDiffStackNodes(new int[] {3,4});
this.setMeanStackNodes(new int[] {3,4});
this.setMinStackNodes(new int[] {3,4});
this.setMaxStackNodes(new int[] {3,4});
this.setControlStackNodes(new int[] {2,2,2});
this.setDoControlSkip(false);
this.setDoMeanSkip(false);
this.setDoMinSkip(false);
this.setDoMaxSkip(false);
this.setDoDiffSkip(false);
this.setDoMainSkip(false);
}
public ComputationGraphConfiguration getConf(){
ComputationGraphConfiguration.GraphBuilder confBuilder = new NeuralNetConfiguration.Builder()
.iterations(1)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.learningRate(getLearningRate())
.updater(Updater.ADAM).adamMeanDecay(1. - 1./10.).adamVarDecay(1. - 1./50.)
.weightInit(WeightInit.XAVIER_UNIFORM)
//.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
.regularization(true).useDropConnect(true)
.l2(getL2()).l1(getL1()).dropOut(getDropout())
.graphBuilder()
.setInputTypes(InputType.recurrent(getNumInputs()),InputType.recurrent(getNumStateInput()))
.addInputs("X","State");
String[] DiffStack = new String[getDepthDiff()];
String[] MeanStack = new String[getDepthMean()];
String[] MaxStack = new String[getDepthMax()];
String[] MinStack = new String[getDepthMin()];
String[] MainStack = new String[getDepthMain()];
String[] ControlStack = new String[getDepthControl()];
ControlStack[0] = "ControlStack-"+0;
confBuilder.addLayer(ControlStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getNumInputs()+getNumStateInput())
.nOut(getControlStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),"X","State");
for(int i = 1; i < getDepthControl(); i++){
ControlStack[i] = "ControlStack-"+i;
if (isDoControlSkip()){
if (i > 1){
confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getControlStackNodes()[i-1] + getControlStackNodes()[0])
.nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), ControlStack[0], ControlStack[i-1]);
} else {
confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getControlStackNodes()[0])
.nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), ControlStack[0]);
}
} else {
confBuilder.addLayer(ControlStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getControlStackNodes()[i-1])
.nOut(getControlStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), ControlStack[i-1]);
}
}
MinStack[0] = "MinStack-"+0;
confBuilder.addLayer(MinStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1])
.nOut(getMinStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),"X",ControlStack[getDepthControl()-1]);
for(int i = 1; i < getDepthMin(); i++){
MinStack[i] = "MinStack-"+i;
if (isDoMinSkip()){
if (i > 1){
confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMinStackNodes()[i-1] + getMinStackNodes()[0])
.nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MinStack[0], MinStack[i-1]);
} else {
confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMinStackNodes()[0])
.nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MinStack[0]);
}
} else {
confBuilder.addLayer(MinStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMinStackNodes()[i-1])
.nOut(getMinStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MinStack[i-1]);
}
}
MaxStack[0] = "MaxStack-"+0;
confBuilder.addLayer(MaxStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1])
.nOut(getMaxStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),"X",ControlStack[getDepthControl()-1]);
for(int i = 1; i < getDepthMax(); i++){
MaxStack[i] = "MaxStack-"+i;
if (isDoMaxSkip()){
if (i > 1){
confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMaxStackNodes()[i-1] + getMaxStackNodes()[0])
.nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MaxStack[0], MaxStack[i-1]);
} else {
confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMaxStackNodes()[0])
.nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MaxStack[0]);
}
} else {
confBuilder.addLayer(MaxStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMaxStackNodes()[i-1])
.nOut(getMaxStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MaxStack[i-1]);
}
}
MeanStack[0] = "MeanStack-"+0;
confBuilder.addLayer(MeanStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getNumInputs() + getControlStackNodes()[getDepthControl()-1])
.nOut(getMeanStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),"X",ControlStack[getDepthControl()-1]);
for(int i = 1; i < getDepthMean(); i++){
MeanStack[i] = "MeanStack-"+i;
if (isDoMeanSkip()){
if (i > 1){
confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMeanStackNodes()[i-1] + getMeanStackNodes()[0])
.nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MeanStack[0], MeanStack[i-1]);
} else {
confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMeanStackNodes()[0])
.nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MeanStack[0]);
}
} else {
confBuilder.addLayer(MeanStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMeanStackNodes()[i-1])
.nOut(getMeanStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MeanStack[i-1]);
}
}
DiffStack[0] = "DiffStack-"+0;
confBuilder.addLayer(DiffStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMinStackNodes()[getDepthMin()-1] + getMaxStackNodes()[getDepthMax()-1] + getControlStackNodes()[getDepthControl()-1])
.nOut(getDiffStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),MinStack[getDepthMin()-1], MaxStack[getDepthMax()-1],ControlStack[getDepthControl()-1]);
for(int i = 1; i < getDepthDiff(); i++){
DiffStack[i] = "DiffStack-"+i;
if (isDoDiffSkip()){
if (i > 1){
confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getDiffStackNodes()[i-1] + getDiffStackNodes()[0])
.nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), DiffStack[0], DiffStack[i-1]);
} else {
confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getDiffStackNodes()[0])
.nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), DiffStack[0]);
}
} else {
confBuilder.addLayer(DiffStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getDiffStackNodes()[i-1])
.nOut(getDiffStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), DiffStack[i-1]);
}
}
MainStack[0] = "MainStack-"+0;
confBuilder.addLayer(MainStack[0], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getNumInputs() + getDiffStackNodes()[getDepthDiff()-1] + getMeanStackNodes()[getDepthMean()-1] + getControlStackNodes()[getDepthControl()-1])
.nOut(getMainStackNodes()[0]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(),"X", DiffStack[getDepthDiff()-1], MeanStack[getDepthMean()-1], ControlStack[getDepthControl()-1]);
for(int i = 1; i < getDepthMain(); i++){
MainStack[i] = "MainStack-"+i;
if (isDoMainSkip()){
if (i > 1){
confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMainStackNodes()[i-1] + getMainStackNodes()[0])
.nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MainStack[0], MainStack[i-1]);
} else {
confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMainStackNodes()[0])
.nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MainStack[0]);
}
} else {
confBuilder.addLayer(MainStack[i], new GravesLSTM.Builder().gateActivationFunction(Activation.SIGMOID)
.nIn(getMainStackNodes()[i-1])
.nOut(getMainStackNodes()[i]).forgetGateBiasInit(5.)
.activation(Activation.TANH)
.build(), MainStack[i-1]);
}
}
// outputs
confBuilder.addLayer("out1", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
.nIn(getMainStackNodes()[getDepthMain()-1])
.nOut(1)
.activation(Activation.IDENTITY)
.build(), MainStack[getDepthMain()-1]);
confBuilder.addLayer("out2", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
.nIn(getMainStackNodes()[getDepthMain()-1])
.nOut(1)
.activation(Activation.IDENTITY)
.build(), MainStack[getDepthMain()-1]);
confBuilder.addLayer("out3", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
.nIn(getMainStackNodes()[getDepthMain()-1])
.nOut(1)
.activation(Activation.IDENTITY)
.build(), MainStack[getDepthMain()-1]);
confBuilder.addLayer("out4", new RnnOutputLayer.Builder(LossFunctions.LossFunction.KL_DIVERGENCE)
.nIn(getMainStackNodes()[getDepthMain()-1])
.nOut(1)
.activation(Activation.SIGMOID)
.build(), MainStack[getDepthMain()-1]);
confBuilder.setOutputs("out1","out2","out3","out4");
ComputationGraphConfiguration cgconf = confBuilder.pretrain(false)
.backprop(true).backpropType(BackpropType.Standard)
//.tBPTTForwardLength(500).tBPTTBackwardLength(500)
.build();
return cgconf;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment