use of org.deeplearning4j.ui.storage.mapdb.MapDBStatsStorage in project deeplearning4j by deeplearning4j.
the class TestStatsListener method testListenerBasic.
@Test
public void testListenerBasic() {
for (boolean useJ7 : new boolean[] { false, true }) {
DataSet ds = new IrisDataSetIterator(150, 150).next();
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(1).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list().layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(4).nOut(3).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
//in-memory
StatsStorage ss = new MapDBStatsStorage();
if (useJ7) {
net.setListeners(new J7StatsListener(ss));
} else {
net.setListeners(new StatsListener(ss));
}
for (int i = 0; i < 3; i++) {
net.fit(ds);
}
List<String> sids = ss.listSessionIDs();
assertEquals(1, sids.size());
String sessionID = ss.listSessionIDs().get(0);
assertEquals(1, ss.listTypeIDsForSession(sessionID).size());
String typeID = ss.listTypeIDsForSession(sessionID).get(0);
assertEquals(1, ss.listWorkerIDsForSession(sessionID).size());
String workerID = ss.listWorkerIDsForSession(sessionID).get(0);
Persistable staticInfo = ss.getStaticInfo(sessionID, typeID, workerID);
assertNotNull(staticInfo);
System.out.println(staticInfo);
List<Persistable> updates = ss.getAllUpdatesAfter(sessionID, typeID, workerID, 0);
assertEquals(3, updates.size());
for (Persistable p : updates) {
System.out.println(p);
}
}
}
use of org.deeplearning4j.ui.storage.mapdb.MapDBStatsStorage in project deeplearning4j by deeplearning4j.
the class TestListeners method testStatsCollection.
@Test
public void testStatsCollection() {
JavaSparkContext sc = getContext();
int nExecutors = numExecutors();
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3).activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork network = new MultiLayerNetwork(conf);
network.init();
TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).batchSizePerWorker(5).averagingFrequency(6).build();
SparkDl4jMultiLayer net = new SparkDl4jMultiLayer(sc, conf, tm);
//In-memory
StatsStorage ss = new MapDBStatsStorage();
net.setListeners(ss, Collections.singletonList(new StatsListener(null)));
List<DataSet> list = new IrisDataSetIterator(120, 150).next().asList();
//120 examples, 4 executors, 30 examples per executor -> 6 updates of size 5 per executor
JavaRDD<DataSet> rdd = sc.parallelize(list);
net.fit(rdd);
List<String> sessions = ss.listSessionIDs();
System.out.println("Sessions: " + sessions);
assertEquals(1, sessions.size());
String sid = sessions.get(0);
List<String> typeIDs = ss.listTypeIDsForSession(sid);
List<String> workers = ss.listWorkerIDsForSession(sid);
System.out.println(sid + "\t" + typeIDs + "\t" + workers);
List<Persistable> lastUpdates = ss.getLatestUpdateAllWorkers(sid, StatsListener.TYPE_ID);
System.out.println(lastUpdates);
System.out.println("Static info:");
for (String wid : workers) {
Persistable staticInfo = ss.getStaticInfo(sid, StatsListener.TYPE_ID, wid);
System.out.println(sid + "\t" + wid);
}
assertEquals(1, typeIDs.size());
assertEquals(numExecutors(), workers.size());
String firstWorker = workers.get(0);
String firstWorkerSubstring = workers.get(0).substring(0, firstWorker.length() - 1);
for (String wid : workers) {
String widSubstring = wid.substring(0, wid.length() - 1);
assertEquals(firstWorkerSubstring, widSubstring);
String counterVal = wid.substring(wid.length() - 1, wid.length());
int cv = Integer.parseInt(counterVal);
assertTrue(0 <= cv && cv < numExecutors());
}
}
Aggregations