Skip to content

Instantly share code, notes, and snippets.

@mindcrime
Created February 26, 2017 18:53
Show Gist options
  • Save mindcrime/6c149827205184617d4edad6f00342f3 to your computer and use it in GitHub Desktop.
Save mindcrime/6c149827205184617d4edad6f00342f3 to your computer and use it in GitHub Desktop.
Attempt to manually use scaler
JavaPairRDD<String, PortableDataStream> origData = sc.binaryFiles("/home/prhodes/development/experimental/ai_exp/NeuralNetworkSandbox/mnist_png/cutdown/0/**");
ImageRecordReader irr = new ImageRecordReader(28, 28, 1, new ParentPathLabelGenerator() );
List<String> labelsList = Arrays.asList( "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" );
irr.setLabels(labelsList);
RecordReaderFunction rrf = new RecordReaderFunction(irr);
JavaRDD<List<Writable>> rdd = origData.map(rrf);
System.out.println( "DataSet RDD created");
DataNormalization scaler = new ImagePreProcessingScaler(0,1);
JavaRDD<DataSet> trainingData = rdd.map(new DataVecDataSetFunction(1,10, false, scaler, null ));
trainingData.foreach( new VoidFunction<DataSet>() {
DataNormalization scaler = new ImagePreProcessingScaler(0,1);
int count = 0;
@Override
public void call(DataSet arg0) throws Exception {
System.out.println( "scaling for count: " + count++ + "\n");
scaler.preProcess(arg0);
}
} );
trainingData.foreach( new VoidFunction<DataSet>() {
int count = 0;
@Override
public void call(DataSet arg0) throws Exception {
System.out.println( "count: " + count++ + "\n");
System.out.println( "features: " + arg0.getFeatures() + "\n");
}
} );
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment