Examples with NotebookOutput com.simiacryptus.util.io.NotebookOutput used on opensource projects

Example 6 with NotebookOutput

use of com.simiacryptus.util.io.NotebookOutput in project MindsEye by SimiaCryptus.

the class ClassifyProblem method run.

@Nonnull
@Override
public ClassifyProblem run(@Nonnull final NotebookOutput log) {
    @Nonnull final TrainingMonitor monitor = TestUtil.getMonitor(history);
    final Tensor[][] trainingData = getTrainingData(log);
    @Nonnull final DAGNetwork network = fwdFactory.imageToVector(log, categories);
    log.h3("Network Diagram");
    log.code(() -> {
        return Graphviz.fromGraph(TestUtil.toGraph(network)).height(400).width(600).render(Format.PNG).toImage();
    });
    log.h3("Training");
    @Nonnull final SimpleLossNetwork supervisedNetwork = new SimpleLossNetwork(network, new EntropyLossLayer());
    TestUtil.instrumentPerformance(supervisedNetwork);
    int initialSampleSize = Math.max(trainingData.length / 5, Math.min(10, trainingData.length / 2));
    @Nonnull final ValidatingTrainer trainer = optimizer.train(log, new SampledArrayTrainable(trainingData, supervisedNetwork, initialSampleSize, getBatchSize()), new ArrayTrainable(trainingData, supervisedNetwork, getBatchSize()), monitor);
    log.code(() -> {
        trainer.setTimeout(timeoutMinutes, TimeUnit.MINUTES).setMaxIterations(10000).run();
    });
    if (!history.isEmpty()) {
        log.code(() -> {
            return TestUtil.plot(history);
        });
        log.code(() -> {
            return TestUtil.plotTime(history);
        });
    }
    try {
        @Nonnull String filename = log.getName() + "_" + ClassifyProblem.modelNo++ + "_plot.png";
        ImageIO.write(Util.toImage(TestUtil.plot(history)), "png", log.file(filename));
        @Nonnull File file = new File(log.getResourceDir(), filename);
        log.appendFrontMatterProperty("result_plot", file.toString(), ";");
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    TestUtil.extractPerformance(log, supervisedNetwork);
    @Nonnull final String modelName = "classification_model_" + ClassifyProblem.modelNo++ + ".json";
    log.appendFrontMatterProperty("result_model", modelName, ";");
    log.p("Saved model as " + log.file(network.getJson().toString(), modelName, modelName));
    log.h3("Validation");
    log.p("If we apply our model against the entire validation dataset, we get this accuracy:");
    log.code(() -> {
        return data.validationData().mapToDouble(labeledObject -> predict(network, labeledObject)[0] == parse(labeledObject.label) ? 1 : 0).average().getAsDouble() * 100;
    });
    log.p("Let's examine some incorrectly predicted results in more detail:");
    log.code(() -> {
        try {
            @Nonnull final TableOutput table = new TableOutput();
            Lists.partition(data.validationData().collect(Collectors.toList()), 100).stream().flatMap(batch -> {
                @Nonnull TensorList batchIn = TensorArray.create(batch.stream().map(x -> x.data).toArray(i -> new Tensor[i]));
                TensorList batchOut = network.eval(new ConstantResult(batchIn)).getData();
                return IntStream.range(0, batchOut.length()).mapToObj(i -> toRow(log, batch.get(i), batchOut.get(i).getData()));
            }).filter(x -> null != x).limit(10).forEach(table::putRow);
            return table;
        } catch (@Nonnull final IOException e) {
            throw new RuntimeException(e);
        }
    });
    return this;
}

Example 7 with NotebookOutput

use of com.simiacryptus.util.io.NotebookOutput in project MindsEye by SimiaCryptus.

the class ImgCropLayerTest method getPerformanceTester.

@Nullable
@Override
public ComponentTest<ToleranceStatistics> getPerformanceTester() {
    @Nonnull ComponentTest<ToleranceStatistics> inner = new PerformanceTester().setSamples(100).setBatches(10);
    return new ComponentTestBase<ToleranceStatistics>() {

        @Override
        public ToleranceStatistics test(@Nonnull NotebookOutput log, Layer component, Tensor... inputPrototype) {
            @Nullable PrintStream apiLog = null;
            try {
                apiLog = new PrintStream(log.file("cuda_perf.log"));
                CudaSystem.addLog(apiLog);
                return inner.test(log, component, inputPrototype);
            } finally {
                log.p(log.file((String) null, "cuda_perf.log", "GPU Log"));
                if (null != apiLog) {
                    apiLog.close();
                    CudaSystem.apiLog.remove(apiLog);
                }
            }
        }

        @Override
        protected void _free() {
            inner.freeRef();
            super._free();
        }
    };
}

Example 8 with NotebookOutput

use of com.simiacryptus.util.io.NotebookOutput in project MindsEye by SimiaCryptus.

the class ActivationLayerTestBase method run.

@Override
public void run(final NotebookOutput log) {
    super.run(log);
    log.h3("Function Plots");
    final Layer layer = getLayer(new int[][] { { 1 } }, new Random());
    final List<double[]> plotData = scan().mapToObj(x -> {
        @Nonnull Tensor tensor = new Tensor(x);
        @Nonnull final SimpleEval eval = SimpleEval.run(layer, tensor);
        tensor.freeRef();
        @Nonnull double[] doubles = { x, eval.getOutput().get(0), eval.getDerivative()[0].get(0) };
        eval.freeRef();
        return doubles;
    }).collect(Collectors.toList());
    log.code(() -> {
        return ActivationLayerTestBase.plot("Value Plot", plotData, x -> new double[] { x[0], x[1] });
    });
    log.code(() -> {
        return ActivationLayerTestBase.plot("Derivative Plot", plotData, x -> new double[] { x[0], x[2] });
    });
}

Example 9 with NotebookOutput

use of com.simiacryptus.util.io.NotebookOutput in project MindsEye by SimiaCryptus.

the class EncodingUtil method decompositionSvg.

/**
 * To svg string.
 *
 * @param log              the log
 * @param baseline         the baseline
 * @param signedComponents the signed components
 * @return the string
 */
public static String decompositionSvg(@Nonnull final NotebookOutput log, @Nonnull final Tensor baseline, @Nonnull final List<Tensor> signedComponents) {
    final List<DoubleStatistics> componentStats = signedComponents.stream().map(t -> new DoubleStatistics().accept(t.getData())).collect(Collectors.toList());
    @Nonnull final CharSequence positiveFilter = IntStream.range(0, signedComponents.size()).mapToObj(i -> {
        String name;
        try {
            name = String.format("component_%s.png", EncodingUtil.imageNumber++);
            ImageIO.write(signedComponents.get(i).map(v -> v > 0 ? v * (0xFF / componentStats.get(i).getMax()) : 0).toImage(), "png", log.file(name));
        } catch (@Nonnull final IOException e) {
            throw new RuntimeException(e);
        }
        return String.format("  <feImage xlink:href=\"%s\" result=\"pos_image_%s\" />\n", name, i);
    }).reduce((a, b) -> a + "\n" + b).get();
    @Nonnull final CharSequence negativeFilter = IntStream.range(0, signedComponents.size()).mapToObj(i -> {
        String name;
        try {
            name = String.format("component_%s.png", EncodingUtil.imageNumber++);
            ImageIO.write(signedComponents.get(i).map(v -> v < 0 ? 0xFF - v * (0xFF / componentStats.get(i).getMin()) : 0).toImage(), "png", log.file(name));
        } catch (@Nonnull final IOException e) {
            throw new RuntimeException(e);
        }
        return String.format("  <feImage xlink:href=\"%s\" result=\"neg_image_%s\" />\n", name, i);
    }).reduce((a, b) -> a + "\n" + b).get();
    @Nonnull final CharSequence compositingFilters = IntStream.range(0, signedComponents.size()).mapToObj(i -> {
        final double fPos = componentStats.get(i).getMax() / 0xFF;
        final double fNeg = componentStats.get(i).getMin() / 0xFF;
        return "  <feComposite in=\"" + (i == 0 ? "FillPaint" : "lastResult") + "\" in2=\"neg_image_" + i + "\" result=\"lastResult\" operator=\"arithmetic\" k1=\"0.0\" k2=\"1.0\" k3=\"" + -fNeg + "\" k4=\"" + fNeg + "\"/>\n" + "  <feComposite in=\"lastResult\" in2=\"pos_image_" + i + "\" result=\"lastResult\" operator=\"arithmetic\" k1=\"0.0\" k2=\"1.0\" k3=\"" + fPos + "\" k4=\"" + 0.0 + "\"/>\n";
    }).reduce((a, b) -> a + "\n" + b).get();
    final int red = (int) baseline.get(0, 0, 0);
    final int green = (int) baseline.get(0, 0, 1);
    final int blue = (int) baseline.get(0, 0, 2);
    @Nonnull final CharSequence avgHexColor = Long.toHexString(red + (green << 8) + (blue << 16));
    return "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n" + ("<defs>\n" + "<filter id=\"image\" >\n" + (positiveFilter + "\n" + negativeFilter + "\n" + compositingFilters).replaceAll("\n", "\n\t") + "\n" + "</filter>\n" + "</defs>\n" + "<rect style=\"filter:url(#image);\" setByCoord=\"#" + avgHexColor + "\" width=\"256\" height=\"256\"/>").replaceAll("\n", "\n\t") + "\n</svg>";
}

Example 10 with NotebookOutput

use of com.simiacryptus.util.io.NotebookOutput in project MindsEye by SimiaCryptus.

the class EncodingUtil method animatedGif.

/**
 * Animated gif string.
 *
 * @param log              the log
 * @param baseline         the baseline
 * @param signedComponents the signed components
 * @return the string
 */
public static CharSequence animatedGif(@Nonnull final NotebookOutput log, @Nonnull final Tensor baseline, @Nonnull final List<Tensor> signedComponents) {
    int loopTimeMs = 15000;
    int framerate = 12;
    int frames = loopTimeMs * framerate / 1000;
    try {
        double step = 2 * Math.PI / frames;
        @Nonnull String filename = EncodingUtil.gifNumber++ + ".gif";
        @Nonnull File file = new File(log.getResourceDir(), filename);
        GifSequenceWriter.write(file, loopTimeMs / frames, true, DoubleStream.iterate(0, x -> x + step).limit(frames).parallel().mapToObj(t -> {
            return IntStream.range(0, signedComponents.size()).mapToObj(i -> {
                return signedComponents.get(i).scale((1 + Math.sin((1 + i) * t)) / 2);
            }).reduce((a, b) -> {
                Tensor add = a.addAndFree(b);
                b.freeRef();
                return add;
            }).get().add(baseline).toImage();
        }).toArray(i -> new BufferedImage[i]));
        return String.format("<img src=\"etc/%s\" />", filename);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}