MyCNNMnistExample experiment 3

MyCNNMnistExample.java

package org.deeplearning4j.examples.convolution;

import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.GradientNormalization;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.layers.ConvolutionLayer;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.conf.layers.SubsamplingLayer;
import org.deeplearning4j.nn.conf.layers.setup.ConvolutionLayerSetup;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.api.IterationListener;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.deeplearning4j.ui.weights.HistogramIterationListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.SplitTestAndTrain;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;

/**
 * Created by willow on 5/11/15.
 */
public class MyCNNMnistExample {

    private static final Logger log = LoggerFactory.getLogger(MyCNNMnistExample.class);

    public static void main(String[] args) throws Exception {
        int numRows = 28;
        int numColumns = 28;
        int nChannels = 1;
        int outputNum = 10;
        int numSamples = 10000;
        int batchSize = 100;
        int iterations = 1;
        int splitTrainNum = (int) (batchSize*.8);
        int seed = 123;
        int listenerFreq = Math.max(iterations/10, 1);
        DataSet mnist;
        SplitTestAndTrain trainTest;
        DataSet trainInput;
        List<INDArray> testInput = new ArrayList<>();
        List<INDArray> testLabels = new ArrayList<>();

        log.info("Load data....");
        DataSetIterator mnistIter = new MnistDataSetIterator(batchSize,numSamples, true);

        log.info("Build model....");
        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                .seed(seed)
                .iterations(iterations)
                //.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .learningRate(0.01) // default
                //.momentum(0.9)
                .regularization(true)
                .list(3)
                .layer(0, new ConvolutionLayer.Builder(3, 3) // 28*28*1 => 28*28*10
                        .nIn(nChannels)
                        .nOut(10)
                        .padding(1, 1)
                        .stride(1, 1)
                        .weightInit(WeightInit.RELU)
                        .activation("relu")
                        .build())
                .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2,2}) // 28*28*10 => 14*14*10
                        .stride(2, 2)
                        .build())
                /*.layer(1, new ConvolutionLayer.Builder(4, 4) // 14*14*10 => 7*7*20
                        .nIn(10)
                        .nOut(20)
                        .padding(2, 2)
                        .stride(2, 2)
                        .weightInit(WeightInit.RELU)
                        .activation("relu")
                        .build())*/
                /*.layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2,2}) // 14*14*20 => 7*7*20
                        .stride(2, 2)
                        .build())*/
                .layer(1, new DenseLayer.Builder().activation("relu")
                        .nOut(200).build())
                .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.RMSE_XENT)
                        .nOut(outputNum)
                        .weightInit(WeightInit.RELU)
                        .activation("softmax")
                        .updater(Updater.SGD)
                        .build())
                .backprop(true).pretrain(false);

        new ConvolutionLayerSetup(builder,numRows,numColumns,nChannels);

        MultiLayerConfiguration conf = builder.build();

        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        log.info("Train model....");
        model.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(listenerFreq), new HistogramIterationListener(listenerFreq)));
        while(mnistIter.hasNext()) {
            mnist = mnistIter.next();
            trainTest = mnist.splitTestAndTrain(splitTrainNum, new Random(seed)); // train set that is the result
            trainInput = trainTest.getTrain(); // get feature matrix and labels for training
            testInput.add(trainTest.getTest().getFeatureMatrix());
            testLabels.add(trainTest.getTest().getLabels());
            model.fit(trainInput);
        }

        log.info("Evaluate weights....");

        log.info("Evaluate model....");
        Evaluation eval = new Evaluation(outputNum);
        for(int i = 0; i < testInput.size(); i++) {
            INDArray output = model.output(testInput.get(i));
            eval.eval(testLabels.get(i), output);
        }
        INDArray output = model.output(testInput.get(0));
        eval.eval(testLabels.get(0), output);
        log.info(eval.stats());
        log.info("****************Example finished********************");


    }
}

score result = 0.60, 0.886

1st CNN nOut = 3

==========================Scores========================================
 Accuracy:  0.5906
 Precision: 0.6348
 Recall:    0.5787
 F1 Score:  0.6054558488467993
===========================================================================

1st CNN nOut = 30

==========================Scores========================================
 Accuracy:  0.8866
 Precision: 0.8903
 Recall:    0.8832
 F1 Score:  0.8867617677750961
===========================================================================

score result = 0.78, 0.83

batchSize = 1000

==========================Scores========================================
 Accuracy:  0.785
 Precision: 0.7857
 Recall:    0.7748
 F1 Score:  0.7801665296274718
===========================================================================

batchSize = 50

==========================Scores========================================
 Accuracy:  0.7721
 Precision: 0.8118
 Recall:    0.8533
 F1 Score:  0.8320270474485852
===========================================================================

score result = 0.874, 0.889

learningRate=0.001, iterations=5

==========================Scores========================================
 Accuracy:  0.8723
 Precision: 0.8791
 Recall:    0.8691
 F1 Score:  0.8740435275958083
===========================================================================

learningRate=0.001, iterations=10

==========================Scores========================================
 Accuracy:  0.8881
 Precision: 0.8925
 Recall:    0.8862
 F1 Score:  0.8893205912421004
===========================================================================

score result = 0.886, 0.885

DenseLayer nOut = 100

==========================Scores========================================
 Accuracy:  0.8851
 Precision: 0.8898
 Recall:    0.8826
 F1 Score:  0.8862156007098383
===========================================================================

DenseLayer nOut = 300

==========================Scores========================================
 Accuracy:  0.8871
 Precision: 0.8922
 Recall:    0.8849
 F1 Score:  0.8885275979828012
===========================================================================

score result = 0.887, 0.885

kernel size = 3, 3

==========================Scores========================================
 Accuracy:  0.8881
 Precision: 0.8909
 Recall:    0.8847
 F1 Score:  0.8878062567206496
===========================================================================

kernel size = 7, 7

==========================Scores========================================
 Accuracy:  0.8837
 Precision: 0.8895
 Recall:    0.8824
 F1 Score:  0.8859275210003372
===========================================================================