use of com.simiacryptus.mindseye.opt.TrainingMonitor in project MindsEye by SimiaCryptus.
the class TrainingTester method trainMagic.
/**
* Train lbfgs list.
*
* @param log the log
* @param trainable the trainable
* @return the list
*/
@Nonnull
public List<StepRecord> trainMagic(@Nonnull final NotebookOutput log, final Trainable trainable) {
log.p("Now we train using an experimental optimizer:");
@Nonnull final List<StepRecord> history = new ArrayList<>();
@Nonnull final TrainingMonitor monitor = TrainingTester.getMonitor(history);
try {
log.code(() -> {
return new IterativeTrainer(trainable).setLineSearchFactory(label -> new StaticLearningRate(1.0)).setOrientation(new RecursiveSubspace() {
@Override
public void train(@Nonnull TrainingMonitor monitor, Layer macroLayer) {
@Nonnull Tensor[][] nullData = { { new Tensor() } };
@Nonnull BasicTrainable inner = new BasicTrainable(macroLayer);
@Nonnull ArrayTrainable trainable1 = new ArrayTrainable(inner, nullData);
inner.freeRef();
new IterativeTrainer(trainable1).setOrientation(new QQN()).setLineSearchFactory(n -> new QuadraticSearch().setCurrentRate(n.equals(QQN.CURSOR_NAME) ? 1.0 : 1e-4)).setMonitor(new TrainingMonitor() {
@Override
public void log(String msg) {
monitor.log("\t" + msg);
}
}).setMaxIterations(getIterations()).setIterationsPerSample(getIterations()).runAndFree();
trainable1.freeRef();
for (@Nonnull Tensor[] tensors : nullData) {
for (@Nonnull Tensor tensor : tensors) {
tensor.freeRef();
}
}
}
}).setMonitor(monitor).setTimeout(30, TimeUnit.SECONDS).setIterationsPerSample(100).setMaxIterations(250).setTerminateThreshold(0).runAndFree();
});
} catch (Throwable e) {
if (isThrowExceptions())
throw new RuntimeException(e);
}
return history;
}
use of com.simiacryptus.mindseye.opt.TrainingMonitor in project MindsEye by SimiaCryptus.
the class AutoencodingProblem method run.
@Nonnull
@Override
public AutoencodingProblem run(@Nonnull final NotebookOutput log) {
@Nonnull final DAGNetwork fwdNetwork = fwdFactory.imageToVector(log, features);
@Nonnull final DAGNetwork revNetwork = revFactory.vectorToImage(log, features);
@Nonnull final PipelineNetwork echoNetwork = new PipelineNetwork(1);
echoNetwork.add(fwdNetwork);
echoNetwork.add(revNetwork);
@Nonnull final PipelineNetwork supervisedNetwork = new PipelineNetwork(1);
supervisedNetwork.add(fwdNetwork);
@Nonnull final DropoutNoiseLayer dropoutNoiseLayer = new DropoutNoiseLayer().setValue(dropout);
supervisedNetwork.add(dropoutNoiseLayer);
supervisedNetwork.add(revNetwork);
supervisedNetwork.add(new MeanSqLossLayer(), supervisedNetwork.getHead(), supervisedNetwork.getInput(0));
log.h3("Network Diagrams");
log.code(() -> {
return Graphviz.fromGraph(TestUtil.toGraph(fwdNetwork)).height(400).width(600).render(Format.PNG).toImage();
});
log.code(() -> {
return Graphviz.fromGraph(TestUtil.toGraph(revNetwork)).height(400).width(600).render(Format.PNG).toImage();
});
log.code(() -> {
return Graphviz.fromGraph(TestUtil.toGraph(supervisedNetwork)).height(400).width(600).render(Format.PNG).toImage();
});
@Nonnull final TrainingMonitor monitor = new TrainingMonitor() {
@Nonnull
TrainingMonitor inner = TestUtil.getMonitor(history);
@Override
public void log(final String msg) {
inner.log(msg);
}
@Override
public void onStepComplete(final Step currentPoint) {
dropoutNoiseLayer.shuffle(StochasticComponent.random.get().nextLong());
inner.onStepComplete(currentPoint);
}
};
final Tensor[][] trainingData = getTrainingData(log);
// MonitoredObject monitoringRoot = new MonitoredObject();
// TestUtil.addMonitoring(supervisedNetwork, monitoringRoot);
log.h3("Training");
TestUtil.instrumentPerformance(supervisedNetwork);
@Nonnull final ValidatingTrainer trainer = optimizer.train(log, new SampledArrayTrainable(trainingData, supervisedNetwork, trainingData.length / 2, batchSize), new ArrayTrainable(trainingData, supervisedNetwork, batchSize), monitor);
log.code(() -> {
trainer.setTimeout(timeoutMinutes, TimeUnit.MINUTES).setMaxIterations(10000).run();
});
if (!history.isEmpty()) {
log.code(() -> {
return TestUtil.plot(history);
});
log.code(() -> {
return TestUtil.plotTime(history);
});
}
TestUtil.extractPerformance(log, supervisedNetwork);
{
@Nonnull final String modelName = "encoder_model" + AutoencodingProblem.modelNo++ + ".json";
log.p("Saved model as " + log.file(fwdNetwork.getJson().toString(), modelName, modelName));
}
@Nonnull final String modelName = "decoder_model" + AutoencodingProblem.modelNo++ + ".json";
log.p("Saved model as " + log.file(revNetwork.getJson().toString(), modelName, modelName));
// log.h3("Metrics");
// log.code(() -> {
// return TestUtil.toFormattedJson(monitoringRoot.getMetrics());
// });
log.h3("Validation");
log.p("Here are some re-encoded examples:");
log.code(() -> {
@Nonnull final TableOutput table = new TableOutput();
data.validationData().map(labeledObject -> {
return toRow(log, labeledObject, echoNetwork.eval(labeledObject.data).getData().get(0).getData());
}).filter(x -> null != x).limit(10).forEach(table::putRow);
return table;
});
log.p("Some rendered unit vectors:");
for (int featureNumber = 0; featureNumber < features; featureNumber++) {
@Nonnull final Tensor input = new Tensor(features).set(featureNumber, 1);
@Nullable final Tensor tensor = revNetwork.eval(input).getData().get(0);
log.out(log.image(tensor.toImage(), ""));
}
return this;
}
use of com.simiacryptus.mindseye.opt.TrainingMonitor in project MindsEye by SimiaCryptus.
the class EncodingProblem method run.
@Nonnull
@Override
public EncodingProblem run(@Nonnull final NotebookOutput log) {
@Nonnull final TrainingMonitor monitor = TestUtil.getMonitor(history);
Tensor[][] trainingData;
try {
trainingData = data.trainingData().map(labeledObject -> {
return new Tensor[] { new Tensor(features).set(this::random), labeledObject.data };
}).toArray(i -> new Tensor[i][]);
} catch (@Nonnull final IOException e) {
throw new RuntimeException(e);
}
@Nonnull final DAGNetwork imageNetwork = revFactory.vectorToImage(log, features);
log.h3("Network Diagram");
log.code(() -> {
return Graphviz.fromGraph(TestUtil.toGraph(imageNetwork)).height(400).width(600).render(Format.PNG).toImage();
});
@Nonnull final PipelineNetwork trainingNetwork = new PipelineNetwork(2);
@Nullable final DAGNode image = trainingNetwork.add(imageNetwork, trainingNetwork.getInput(0));
@Nullable final DAGNode softmax = trainingNetwork.add(new SoftmaxActivationLayer(), trainingNetwork.getInput(0));
trainingNetwork.add(new SumInputsLayer(), trainingNetwork.add(new EntropyLossLayer(), softmax, softmax), trainingNetwork.add(new NthPowerActivationLayer().setPower(1.0 / 2.0), trainingNetwork.add(new MeanSqLossLayer(), image, trainingNetwork.getInput(1))));
log.h3("Training");
log.p("We start by training apply a very small population to improve initial convergence performance:");
TestUtil.instrumentPerformance(trainingNetwork);
@Nonnull final Tensor[][] primingData = Arrays.copyOfRange(trainingData, 0, 1000);
@Nonnull final ValidatingTrainer preTrainer = optimizer.train(log, (SampledTrainable) new SampledArrayTrainable(primingData, trainingNetwork, trainingSize, batchSize).setMinSamples(trainingSize).setMask(true, false), new ArrayTrainable(primingData, trainingNetwork, batchSize), monitor);
log.code(() -> {
preTrainer.setTimeout(timeoutMinutes / 2, TimeUnit.MINUTES).setMaxIterations(batchSize).run();
});
TestUtil.extractPerformance(log, trainingNetwork);
log.p("Then our main training phase:");
TestUtil.instrumentPerformance(trainingNetwork);
@Nonnull final ValidatingTrainer mainTrainer = optimizer.train(log, (SampledTrainable) new SampledArrayTrainable(trainingData, trainingNetwork, trainingSize, batchSize).setMinSamples(trainingSize).setMask(true, false), new ArrayTrainable(trainingData, trainingNetwork, batchSize), monitor);
log.code(() -> {
mainTrainer.setTimeout(timeoutMinutes, TimeUnit.MINUTES).setMaxIterations(batchSize).run();
});
TestUtil.extractPerformance(log, trainingNetwork);
if (!history.isEmpty()) {
log.code(() -> {
return TestUtil.plot(history);
});
log.code(() -> {
return TestUtil.plotTime(history);
});
}
try {
@Nonnull String filename = log.getName().toString() + EncodingProblem.modelNo++ + "_plot.png";
ImageIO.write(Util.toImage(TestUtil.plot(history)), "png", log.file(filename));
log.appendFrontMatterProperty("result_plot", filename, ";");
} catch (IOException e) {
throw new RuntimeException(e);
}
// log.file()
@Nonnull final String modelName = "encoding_model_" + EncodingProblem.modelNo++ + ".json";
log.appendFrontMatterProperty("result_model", modelName, ";");
log.p("Saved model as " + log.file(trainingNetwork.getJson().toString(), modelName, modelName));
log.h3("Results");
@Nonnull final PipelineNetwork testNetwork = new PipelineNetwork(2);
testNetwork.add(imageNetwork, testNetwork.getInput(0));
log.code(() -> {
@Nonnull final TableOutput table = new TableOutput();
Arrays.stream(trainingData).map(tensorArray -> {
@Nullable final Tensor predictionSignal = testNetwork.eval(tensorArray).getData().get(0);
@Nonnull final LinkedHashMap<CharSequence, Object> row = new LinkedHashMap<>();
row.put("Source", log.image(tensorArray[1].toImage(), ""));
row.put("Echo", log.image(predictionSignal.toImage(), ""));
return row;
}).filter(x -> null != x).limit(10).forEach(table::putRow);
return table;
});
log.p("Learned Model Statistics:");
log.code(() -> {
@Nonnull final ScalarStatistics scalarStatistics = new ScalarStatistics();
trainingNetwork.state().stream().flatMapToDouble(x -> Arrays.stream(x)).forEach(v -> scalarStatistics.add(v));
return scalarStatistics.getMetrics();
});
log.p("Learned Representation Statistics:");
log.code(() -> {
@Nonnull final ScalarStatistics scalarStatistics = new ScalarStatistics();
Arrays.stream(trainingData).flatMapToDouble(row -> Arrays.stream(row[0].getData())).forEach(v -> scalarStatistics.add(v));
return scalarStatistics.getMetrics();
});
log.p("Some rendered unit vectors:");
for (int featureNumber = 0; featureNumber < features; featureNumber++) {
@Nonnull final Tensor input = new Tensor(features).set(featureNumber, 1);
@Nullable final Tensor tensor = imageNetwork.eval(input).getData().get(0);
TestUtil.renderToImages(tensor, true).forEach(img -> {
log.out(log.image(img, ""));
});
}
return this;
}
use of com.simiacryptus.mindseye.opt.TrainingMonitor in project MindsEye by SimiaCryptus.
the class AutoencoderNetwork method train.
/**
* Train autoencoder network . training parameters.
*
* @return the autoencoder network . training parameters
*/
@Nonnull
public AutoencoderNetwork.TrainingParameters train() {
return new AutoencoderNetwork.TrainingParameters() {
@Nonnull
@Override
public SimpleLossNetwork getTrainingNetwork() {
@Nonnull final PipelineNetwork student = new PipelineNetwork();
student.add(encoder);
student.add(decoder);
return new SimpleLossNetwork(student, new MeanSqLossLayer());
}
@Nonnull
@Override
protected TrainingMonitor wrap(@Nonnull final TrainingMonitor monitor) {
return new TrainingMonitor() {
@Override
public void log(final String msg) {
monitor.log(msg);
}
@Override
public void onStepComplete(final Step currentPoint) {
inputNoise.shuffle();
encodedNoise.shuffle(StochasticComponent.random.get().nextLong());
monitor.onStepComplete(currentPoint);
}
};
}
};
}
use of com.simiacryptus.mindseye.opt.TrainingMonitor in project MindsEye by SimiaCryptus.
the class QuantifyOrientationWrapper method orient.
@Override
public LineSearchCursor orient(final Trainable subject, final PointSample measurement, @Nonnull final TrainingMonitor monitor) {
final LineSearchCursor cursor = inner.orient(subject, measurement, monitor);
if (cursor instanceof SimpleLineSearchCursor) {
final DeltaSet<Layer> direction = ((SimpleLineSearchCursor) cursor).direction;
@Nonnull final StateSet<Layer> weights = ((SimpleLineSearchCursor) cursor).origin.weights;
final Map<CharSequence, CharSequence> dataMap = weights.stream().collect(Collectors.groupingBy(x -> getId(x), Collectors.toList())).entrySet().stream().collect(Collectors.toMap(x -> x.getKey(), list -> {
final List<Double> doubleList = list.getValue().stream().map(weightDelta -> {
final DoubleBuffer<Layer> dirDelta = direction.getMap().get(weightDelta.layer);
final double denominator = weightDelta.deltaStatistics().rms();
final double numerator = null == dirDelta ? 0 : dirDelta.deltaStatistics().rms();
return numerator / (0 == denominator ? 1 : denominator);
}).collect(Collectors.toList());
if (1 == doubleList.size())
return Double.toString(doubleList.get(0));
return new DoubleStatistics().accept(doubleList.stream().mapToDouble(x -> x).toArray()).toString();
}));
monitor.log(String.format("Line search stats: %s", dataMap));
} else {
monitor.log(String.format("Non-simple cursor: %s", cursor));
}
return cursor;
}
Aggregations