Examples with SparkDl4jMultiLayer org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer used on opensource projects

Example 16 with SparkDl4jMultiLayer

use of org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testFitViaStringPathsSize1.

@Test
public void testFitViaStringPathsSize1() throws Exception {
    Path tempDir = Files.createTempDirectory("DL4J-testFitViaStringPathsSize1");
    File tempDirF = tempDir.toFile();
    tempDirF.deleteOnExit();
    int dataSetObjSize = 1;
    int batchSizePerExecutor = 25;
    int numSplits = 10;
    int averagingFrequency = 3;
    int totalExamples = numExecutors() * batchSizePerExecutor * numSplits * averagingFrequency;
    DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, totalExamples, false);
    int i = 0;
    while (iter.hasNext()) {
        File nextFile = new File(tempDirF, i + ".bin");
        DataSet ds = iter.next();
        ds.save(nextFile);
        i++;
    }
    System.out.println("Saved to: " + tempDirF.getAbsolutePath());
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize).workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor).averagingFrequency(averagingFrequency).repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);
    //List files:
    Configuration config = new Configuration();
    FileSystem hdfs = FileSystem.get(tempDir.toUri(), config);
    RemoteIterator<LocatedFileStatus> fileIter = hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false);
    List<String> paths = new ArrayList<>();
    while (fileIter.hasNext()) {
        String path = fileIter.next().getPath().toString();
        paths.add(path);
    }
    INDArray paramsBefore = sparkNet.getNetwork().params().dup();
    JavaRDD<String> pathRdd = sc.parallelize(paths);
    sparkNet.fitPaths(pathRdd);
    INDArray paramsAfter = sparkNet.getNetwork().params().dup();
    assertNotEquals(paramsBefore, paramsAfter);
    Thread.sleep(2000);
    SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
    //Expect
    System.out.println(stats.statsAsString());
    assertEquals(numSplits, stats.getValue("ParameterAveragingMasterRepartitionTimesMs").size());
    List<EventStats> list = stats.getValue("ParameterAveragingWorkerFitTimesMs");
    assertEquals(numSplits * numExecutors() * averagingFrequency, list.size());
    for (EventStats es : list) {
        ExampleCountEventStats e = (ExampleCountEventStats) es;
        assertTrue(batchSizePerExecutor * averagingFrequency - 10 >= e.getTotalExampleCount());
    }
    sparkNet.getTrainingMaster().deleteTempFiles(sc);
}

Example 17 with SparkDl4jMultiLayer

use of org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testParameterAveragingMultipleExamplesPerDataSet.

@Test
public void testParameterAveragingMultipleExamplesPerDataSet() throws Exception {
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 25;
    List<DataSet> list = new ArrayList<>();
    DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false);
    while (iter.hasNext()) {
        list.add(iter.next());
    }
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize).batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1).repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);
    JavaRDD<DataSet> rdd = sc.parallelize(list);
    sparkNet.fit(rdd);
    SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
    List<EventStats> mapPartitionStats = stats.getValue("ParameterAveragingMasterMapPartitionsTimesMs");
    //For an averaging frequency of 1
    int numSplits = list.size() * dataSetObjSize / (numExecutors() * batchSizePerExecutor);
    assertEquals(numSplits, mapPartitionStats.size());
    List<EventStats> workerFitStats = stats.getValue("ParameterAveragingWorkerFitTimesMs");
    for (EventStats e : workerFitStats) {
        ExampleCountEventStats eces = (ExampleCountEventStats) e;
        System.out.println(eces.getTotalExampleCount());
    }
    for (EventStats e : workerFitStats) {
        ExampleCountEventStats eces = (ExampleCountEventStats) e;
        assertEquals(batchSizePerExecutor, eces.getTotalExampleCount());
    }
}

Example 18 with SparkDl4jMultiLayer

use of org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testROCMultiClass.

@Test
public void testROCMultiClass() {
    int nArrays = 100;
    int minibatch = 64;
    int steps = 20;
    int nIn = 5;
    int nOut = 3;
    int layerSize = 10;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).build()).layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);
    ROCMultiClass local = new ROCMultiClass(steps);
    List<DataSet> dsList = new ArrayList<>();
    for (int i = 0; i < nArrays; i++) {
        INDArray features = Nd4j.rand(minibatch, nIn);
        INDArray p = net.output(features);
        INDArray l = Nd4j.zeros(minibatch, nOut);
        for (int j = 0; j < minibatch; j++) {
            l.putScalar(j, r.nextInt(nOut), 1.0);
        }
        local.eval(l, p);
        dsList.add(new DataSet(features, l));
    }
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, net, null);
    JavaRDD<DataSet> rdd = sc.parallelize(dsList);
    ROCMultiClass sparkROC = sparkNet.evaluateROCMultiClass(rdd, steps, 32);
    for (int i = 0; i < nOut; i++) {
        assertEquals(sparkROC.calculateAUC(i), sparkROC.calculateAUC(i), 1e-6);
        double[][] arrLocal = local.getResultsAsArray(i);
        double[][] arrSpark = sparkROC.getResultsAsArray(i);
        assertArrayEquals(arrLocal[0], arrSpark[0], 1e-6);
        assertArrayEquals(arrLocal[1], arrSpark[1], 1e-6);
    }
}

Example 19 with SparkDl4jMultiLayer

use of org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testFitViaStringPaths.

@Test
public void testFitViaStringPaths() throws Exception {
    Path tempDir = Files.createTempDirectory("DL4J-testFitViaStringPaths");
    File tempDirF = tempDir.toFile();
    tempDirF.deleteOnExit();
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 25;
    DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false);
    int i = 0;
    while (iter.hasNext()) {
        File nextFile = new File(tempDirF, i + ".bin");
        DataSet ds = iter.next();
        ds.save(nextFile);
        i++;
    }
    System.out.println("Saved to: " + tempDirF.getAbsolutePath());
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize).workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1).repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);
    //List files:
    Configuration config = new Configuration();
    FileSystem hdfs = FileSystem.get(tempDir.toUri(), config);
    RemoteIterator<LocatedFileStatus> fileIter = hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false);
    List<String> paths = new ArrayList<>();
    while (fileIter.hasNext()) {
        String path = fileIter.next().getPath().toString();
        paths.add(path);
    }
    INDArray paramsBefore = sparkNet.getNetwork().params().dup();
    JavaRDD<String> pathRdd = sc.parallelize(paths);
    sparkNet.fitPaths(pathRdd);
    INDArray paramsAfter = sparkNet.getNetwork().params().dup();
    assertNotEquals(paramsBefore, paramsAfter);
    SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
    System.out.println(stats.statsAsString());
    sparkNet.getTrainingMaster().deleteTempFiles(sc);
}

Example 20 with SparkDl4jMultiLayer

use of org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer in project deeplearning4j by deeplearning4j.

the class TestCustomLayer method testSparkWithCustomLayer.

@Test
public void testSparkWithCustomLayer() {
    //Basic test - checks whether exceptions etc are thrown with custom layers + spark
    //Custom layers are tested more extensively in dl4j core
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(0.1).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new CustomLayer(3.14159)).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10).nOut(10).build()).pretrain(false).backprop(true).build();
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).averagingFrequency(2).batchSizePerWorker(5).saveUpdater(true).workerPrefetchNumBatches(0).build();
    SparkDl4jMultiLayer net = new SparkDl4jMultiLayer(sc, conf, tm);
    List<DataSet> testData = new ArrayList<>();
    Random r = new Random(12345);
    for (int i = 0; i < 200; i++) {
        INDArray f = Nd4j.rand(1, 10);
        INDArray l = Nd4j.zeros(1, 10);
        l.putScalar(0, r.nextInt(10), 1.0);
        testData.add(new DataSet(f, l));
    }
    JavaRDD<DataSet> rdd = sc.parallelize(testData);
    net.fit(rdd);
}