Examples with ExampleCountEventStats org.deeplearning4j.spark.stats.ExampleCountEventStats used on opensource projects

Example 1 with ExampleCountEventStats

use of org.deeplearning4j.spark.stats.ExampleCountEventStats in project deeplearning4j by deeplearning4j.

the class StatsCalculationHelper method build.

public CommonSparkTrainingStats build(SparkTrainingStats masterSpecificStats) {
    List<EventStats> totalTime = new ArrayList<>();
    totalTime.add(new ExampleCountEventStats(methodStartTime, returnTime - methodStartTime, totalExampleCount));
    List<EventStats> initTime = new ArrayList<>();
    initTime.add(new BaseEventStats(initalModelBefore, initialModelAfter - initalModelBefore));
    return new CommonSparkTrainingStats.Builder().trainingMasterSpecificStats(masterSpecificStats).workerFlatMapTotalTimeMs(totalTime).workerFlatMapGetInitialModelTimeMs(initTime).workerFlatMapDataSetGetTimesMs(dataSetGetTimes).workerFlatMapProcessMiniBatchTimesMs(processMiniBatchTimes).build();
}

Example 2 with ExampleCountEventStats

use of org.deeplearning4j.spark.stats.ExampleCountEventStats in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testFitViaStringPathsSize1.

@Test
public void testFitViaStringPathsSize1() throws Exception {
    Path tempDir = Files.createTempDirectory("DL4J-testFitViaStringPathsSize1");
    File tempDirF = tempDir.toFile();
    tempDirF.deleteOnExit();
    int dataSetObjSize = 1;
    int batchSizePerExecutor = 25;
    int numSplits = 10;
    int averagingFrequency = 3;
    int totalExamples = numExecutors() * batchSizePerExecutor * numSplits * averagingFrequency;
    DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, totalExamples, false);
    int i = 0;
    while (iter.hasNext()) {
        File nextFile = new File(tempDirF, i + ".bin");
        DataSet ds = iter.next();
        ds.save(nextFile);
        i++;
    }
    System.out.println("Saved to: " + tempDirF.getAbsolutePath());
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize).workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor).averagingFrequency(averagingFrequency).repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);
    //List files:
    Configuration config = new Configuration();
    FileSystem hdfs = FileSystem.get(tempDir.toUri(), config);
    RemoteIterator<LocatedFileStatus> fileIter = hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false);
    List<String> paths = new ArrayList<>();
    while (fileIter.hasNext()) {
        String path = fileIter.next().getPath().toString();
        paths.add(path);
    }
    INDArray paramsBefore = sparkNet.getNetwork().params().dup();
    JavaRDD<String> pathRdd = sc.parallelize(paths);
    sparkNet.fitPaths(pathRdd);
    INDArray paramsAfter = sparkNet.getNetwork().params().dup();
    assertNotEquals(paramsBefore, paramsAfter);
    Thread.sleep(2000);
    SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
    //Expect
    System.out.println(stats.statsAsString());
    assertEquals(numSplits, stats.getValue("ParameterAveragingMasterRepartitionTimesMs").size());
    List<EventStats> list = stats.getValue("ParameterAveragingWorkerFitTimesMs");
    assertEquals(numSplits * numExecutors() * averagingFrequency, list.size());
    for (EventStats es : list) {
        ExampleCountEventStats e = (ExampleCountEventStats) es;
        assertTrue(batchSizePerExecutor * averagingFrequency - 10 >= e.getTotalExampleCount());
    }
    sparkNet.getTrainingMaster().deleteTempFiles(sc);
}

Example 3 with ExampleCountEventStats

use of org.deeplearning4j.spark.stats.ExampleCountEventStats in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testParameterAveragingMultipleExamplesPerDataSet.

@Test
public void testParameterAveragingMultipleExamplesPerDataSet() throws Exception {
    int dataSetObjSize = 5;
    int batchSizePerExecutor = 25;
    List<DataSet> list = new ArrayList<>();
    DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false);
    while (iter.hasNext()) {
        list.add(iter.next());
    }
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize).batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1).repartionData(Repartition.Always).build());
    sparkNet.setCollectTrainingStats(true);
    JavaRDD<DataSet> rdd = sc.parallelize(list);
    sparkNet.fit(rdd);
    SparkTrainingStats stats = sparkNet.getSparkTrainingStats();
    List<EventStats> mapPartitionStats = stats.getValue("ParameterAveragingMasterMapPartitionsTimesMs");
    //For an averaging frequency of 1
    int numSplits = list.size() * dataSetObjSize / (numExecutors() * batchSizePerExecutor);
    assertEquals(numSplits, mapPartitionStats.size());
    List<EventStats> workerFitStats = stats.getValue("ParameterAveragingWorkerFitTimesMs");
    for (EventStats e : workerFitStats) {
        ExampleCountEventStats eces = (ExampleCountEventStats) e;
        System.out.println(eces.getTotalExampleCount());
    }
    for (EventStats e : workerFitStats) {
        ExampleCountEventStats eces = (ExampleCountEventStats) e;
        assertEquals(batchSizePerExecutor, eces.getTotalExampleCount());
    }
}