Examples with ComputationGraph org.deeplearning4j.nn.graph.ComputationGraph used on opensource projects

Example 51 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class FlowIterationListenerTest method setUp.

@Before
public void setUp() throws Exception {
    if (graph == null) {
        int VOCAB_SIZE = 1000;
        ComputationGraphConfiguration configuration = new NeuralNetConfiguration.Builder().regularization(true).l2(0.0001).weightInit(WeightInit.XAVIER).learningRate(0.01).updater(Updater.RMSPROP).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).graphBuilder().addInputs("inEn", "inFr").setInputTypes(InputType.recurrent(VOCAB_SIZE + 1), InputType.recurrent(VOCAB_SIZE + 1)).addLayer("embeddingEn", new EmbeddingLayer.Builder().nIn(VOCAB_SIZE + 1).nOut(128).activation(Activation.IDENTITY).build(), "inEn").addLayer("encoder", new GravesLSTM.Builder().nIn(128).nOut(256).activation(Activation.SOFTSIGN).build(), "embeddingEn").addVertex("lastTimeStep", new LastTimeStepVertex("inEn"), "encoder").addVertex("duplicateTimeStep", new DuplicateToTimeSeriesVertex("inFr"), "lastTimeStep").addLayer("embeddingFr", new EmbeddingLayer.Builder().nIn(VOCAB_SIZE + 1).nOut(128).activation(Activation.IDENTITY).build(), "inFr").addVertex("embeddingFrSeq", new PreprocessorVertex(new FeedForwardToRnnPreProcessor()), "embeddingFr").addLayer("decoder", new GravesLSTM.Builder().nIn(128 + 256).nOut(256).activation(Activation.SOFTSIGN).build(), "embeddingFrSeq", "duplicateTimeStep").addLayer("output", new RnnOutputLayer.Builder().nIn(256).nOut(VOCAB_SIZE + 1).activation(Activation.SOFTMAX).build(), "decoder").setOutputs("output").pretrain(false).backprop(true).build();
        graph = new ComputationGraph(configuration);
        graph.init();
        INDArray input = Nd4j.zeros(10, VOCAB_SIZE, 20);
        graph.setInputs(input, input);
    }
    if (network == null) {
        final int numRows = 40;
        final int numColumns = 40;
        int nChannels = 3;
        int outputNum = LFWLoader.NUM_LABELS;
        int numSamples = LFWLoader.NUM_IMAGES;
        boolean useSubset = false;
        // numSamples/10;
        int batchSize = 200;
        int iterations = 5;
        int splitTrainNum = (int) (batchSize * .8);
        int seed = 123;
        int listenerFreq = iterations / 5;
        DataSet lfwNext;
        SplitTestAndTrain trainTest;
        DataSet trainInput;
        List<INDArray> testInput = new ArrayList<>();
        List<INDArray> testLabels = new ArrayList<>();
        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations).activation(Activation.RELU).weightInit(WeightInit.XAVIER).gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(0.01).momentum(0.9).regularization(true).updater(Updater.ADAGRAD).useDropConnect(true).list().layer(0, new ConvolutionLayer.Builder(4, 4).name("cnn1").nIn(nChannels).stride(1, 1).nOut(20).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).name("pool1").build()).layer(2, new ConvolutionLayer.Builder(3, 3).name("cnn2").stride(1, 1).nOut(40).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).name("pool2").build()).layer(4, new ConvolutionLayer.Builder(3, 3).name("cnn3").stride(1, 1).nOut(60).build()).layer(5, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).name("pool3").build()).layer(6, new ConvolutionLayer.Builder(2, 2).name("cnn4").stride(1, 1).nOut(80).build()).layer(7, new DenseLayer.Builder().name("ffn1").nOut(160).dropOut(0.5).build()).layer(8, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).activation(Activation.SOFTMAX).build()).backprop(true).pretrain(false);
        new ConvolutionLayerSetup(builder, numRows, numColumns, nChannels);
        network = new MultiLayerNetwork(builder.build());
        network.init();
        INDArray input = Nd4j.zeros(10, nChannels, numRows, numColumns);
        network.setInput(input);
    }
}

Example 52 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class ConvolutionalIterationListener method iterationDone.

/**
     * Event listener for each iteration
     *
     * @param model     the model iterating
     * @param iteration the iteration number
     */
@Override
public void iterationDone(Model model, int iteration) {
    if (iteration % freq == 0) {
        List<INDArray> tensors = new ArrayList<>();
        int cnt = 0;
        Random rnd = new Random();
        BufferedImage sourceImage = null;
        if (model instanceof MultiLayerNetwork) {
            MultiLayerNetwork l = (MultiLayerNetwork) model;
            for (Layer layer : l.getLayers()) {
                if (layer.type() == Layer.Type.CONVOLUTIONAL) {
                    INDArray output = layer.activate();
                    int sampleDim = rnd.nextInt(output.shape()[0] - 1) + 1;
                    if (cnt == 0) {
                        INDArray inputs = ((ConvolutionLayer) layer).input();
                        try {
                            sourceImage = restoreRGBImage(inputs.tensorAlongDimension(sampleDim, new int[] { 3, 2, 1 }));
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                    }
                    INDArray tad = output.tensorAlongDimension(sampleDim, 3, 2, 1);
                    tensors.add(tad);
                    cnt++;
                }
            }
        } else if (model instanceof ComputationGraph) {
            ComputationGraph l = (ComputationGraph) model;
            for (Layer layer : l.getLayers()) {
                if (layer.type() == Layer.Type.CONVOLUTIONAL) {
                    INDArray output = layer.activate();
                    int sampleDim = rnd.nextInt(output.shape()[0] - 1) + 1;
                    if (cnt == 0) {
                        INDArray inputs = ((ConvolutionLayer) layer).input();
                        try {
                            sourceImage = restoreRGBImage(inputs.tensorAlongDimension(sampleDim, new int[] { 3, 2, 1 }));
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                    }
                    INDArray tad = output.tensorAlongDimension(sampleDim, 3, 2, 1);
                    tensors.add(tad);
                    cnt++;
                }
            }
        }
        BufferedImage render = rasterizeConvoLayers(tensors, sourceImage);
        Persistable p = new ConvolutionListenerPersistable(sessionID, workerID, System.currentTimeMillis(), render);
        ssr.putStaticInfo(p);
        minibatchNum++;
    }
}

Example 53 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestPlayUI method testUICompGraph.

@Test
@Ignore
public void testUICompGraph() throws Exception {
    StatsStorage ss = new InMemoryStatsStorage();
    UIServer uiServer = UIServer.getInstance();
    uiServer.attach(ss);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addLayer("L0", new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build(), "in").addLayer("L1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(4).nOut(3).build(), "L0").pretrain(false).backprop(true).setOutputs("L1").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.init();
    net.setListeners(new StatsListener(ss), new ScoreIterationListener(1));
    DataSetIterator iter = new IrisDataSetIterator(150, 150);
    for (int i = 0; i < 100; i++) {
        net.fit(iter);
        Thread.sleep(100);
    }
    Thread.sleep(100000);
}

Example 54 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testNoImprovementNEpochsTermination.

@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).learningRate(0.0).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(100), new ScoreImprovementEpochTerminationCondition(5)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS), //Initial score is ~2.5
    new MaxScoreIterationTerminationCondition(7.5)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();
    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}

Example 55 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestEarlyStoppingCompGraph method testEarlyStoppingIris.

@Test
public void testEarlyStoppingIris() {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in").setOutputs("0").pretrain(false).backprop(true).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));
    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>().epochTerminationConditions(new MaxEpochsTerminationCondition(5)).iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES)).scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();
    IEarlyStoppingTrainer<ComputationGraph> trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult<ComputationGraph> result = trainer.fit();
    System.out.println(result);
    assertEquals(5, result.getTotalEpochs());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    Map<Integer, Double> scoreVsIter = result.getScoreVsEpoch();
    assertEquals(5, scoreVsIter.size());
    String expDetails = esConf.getEpochTerminationConditions().get(0).toString();
    assertEquals(expDetails, result.getTerminationDetails());
    ComputationGraph out = result.getBestModel();
    assertNotNull(out);
    //Check that best score actually matches (returned model vs. manually calculated score)
    ComputationGraph bestNetwork = result.getBestModel();
    irisIter.reset();
    double score = bestNetwork.score(irisIter.next());
    assertEquals(result.getBestModelScore(), score, 1e-2);
}