Examples with MultiLayerNetwork org.deeplearning4j.nn.multilayer.MultiLayerNetwork used on opensource projects

Example 6 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class GlobalPoolingMaskingTests method testMaskingCnnDim3_SingleExample.

@Test
public void testMaskingCnnDim3_SingleExample() {
    //Test masking, where mask is along dimension 3
    int minibatch = 1;
    int depthIn = 2;
    int depthOut = 2;
    int nOut = 2;
    int height = 3;
    int width = 6;
    PoolingType[] poolingTypes = new PoolingType[] { PoolingType.SUM, PoolingType.AVG, PoolingType.MAX, PoolingType.PNORM };
    for (PoolingType pt : poolingTypes) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).convolutionMode(ConvolutionMode.Same).seed(12345L).list().layer(0, new ConvolutionLayer.Builder().nIn(depthIn).nOut(depthOut).kernelSize(height, 2).stride(height, 1).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder().poolingType(pt).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(depthOut).nOut(nOut).build()).pretrain(false).backprop(true).build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        INDArray inToBeMasked = Nd4j.rand(new int[] { minibatch, depthIn, height, width });
        //Shape for mask: [minibatch, width]
        INDArray maskArray = Nd4j.create(new double[] { 1, 1, 1, 1, 1, 0 });
        //Multiply the input by the mask array, to ensure the 0s in the mask correspond to 0s in the input vector
        // as would be the case in practice...
        Nd4j.getExecutioner().exec(new BroadcastMulOp(inToBeMasked, maskArray, inToBeMasked, 0, 3));
        net.setLayerMaskArrays(maskArray, null);
        INDArray outMasked = net.output(inToBeMasked);
        net.clearLayerMaskArrays();
        int numSteps = width - 1;
        INDArray subset = inToBeMasked.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, numSteps));
        assertArrayEquals(new int[] { 1, depthIn, height, 5 }, subset.shape());
        INDArray outSubset = net.output(subset);
        INDArray outMaskedSubset = outMasked.getRow(0);
        assertEquals(outSubset, outMaskedSubset);
        //Finally: check gradient calc for exceptions
        net.setLayerMaskArrays(maskArray, null);
        net.setInput(inToBeMasked);
        INDArray labels = Nd4j.create(new double[] { 0, 1 });
        net.setLabels(labels);
        net.computeGradientAndScore();
    }
}

Example 7 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class GravesBidirectionalLSTMTest method testConvergence.

@Test
@Ignore
public void testConvergence() {
    Nd4j.getRandom().setSeed(12345);
    final int state1Len = 100;
    final int state2Len = 30;
    //segment by signal mean
    //Data: has shape [miniBatchSize,nIn,timeSeriesLength];
    final INDArray sig1 = Nd4j.randn(new int[] { 1, 2, state1Len }).mul(0.1);
    final INDArray sig2 = Nd4j.randn(new int[] { 1, 2, state2Len }).mul(0.1).add(Nd4j.ones(new int[] { 1, 2, state2Len }).mul(1.0));
    INDArray sig = Nd4j.concat(2, sig1, sig2);
    INDArray labels = Nd4j.zeros(new int[] { 1, 2, state1Len + state2Len });
    for (int t = 0; t < state1Len; t++) {
        labels.putScalar(new int[] { 0, 0, t }, 1.0);
    }
    for (int t = state1Len; t < state1Len + state2Len; t++) {
        labels.putScalar(new int[] { 0, 1, t }, 1.0);
    }
    for (int i = 0; i < 3; i++) {
        sig = Nd4j.concat(2, sig, sig);
        labels = Nd4j.concat(2, labels, labels);
    }
    final DataSet ds = new DataSet(sig, labels);
    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(5).learningRate(0.1).rmsDecay(0.95).regularization(true).l2(0.001).updater(Updater.ADAGRAD).seed(12345).list().pretrain(false).layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(2).activation(Activation.TANH).build()).backprop(true).build();
    final MultiLayerNetwork net = new MultiLayerNetwork(conf);
    final IterationListener scoreSaver = new IterationListener() {

        @Override
        public boolean invoked() {
            return false;
        }

        @Override
        public void invoke() {
        }

        @Override
        public void iterationDone(Model model, int iteration) {
            score = model.score();
        }
    };
    net.setListeners(scoreSaver, new ScoreIterationListener(1));
    double oldScore = Double.POSITIVE_INFINITY;
    net.init();
    for (int iEpoch = 0; iEpoch < 3; iEpoch++) {
        net.fit(ds);
        System.out.print(String.format("score is %f%n", score));
        assertTrue(!Double.isNaN(score));
        assertTrue(score < 0.9 * oldScore);
        oldScore = score;
        final INDArray output = net.output(ds.getFeatureMatrix());
        Evaluation evaluation = new Evaluation();
        evaluation.evalTimeSeries(ds.getLabels(), output);
        System.out.print(evaluation.stats() + "\n");
    }
}

Example 8 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class GravesLSTMTest method testSingleExample.

@Test
public void testSingleExample() {
    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.SGD).learningRate(0.1).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(1).activation(Activation.TANH).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    INDArray in1 = Nd4j.rand(new int[] { 1, 2, 4 });
    INDArray in2 = Nd4j.rand(new int[] { 1, 2, 5 });
    in2.put(new INDArrayIndex[] { NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, in1);
    assertEquals(in1, in2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4)));
    INDArray labels1 = Nd4j.rand(new int[] { 1, 1, 4 });
    INDArray labels2 = Nd4j.create(1, 1, 5);
    labels2.put(new INDArrayIndex[] { NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, labels1);
    assertEquals(labels1, labels2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4)));
    INDArray out1 = net.output(in1);
    INDArray out2 = net.output(in2);
    System.out.println(Arrays.toString(net.output(in1).data().asFloat()));
    System.out.println(Arrays.toString(net.output(in2).data().asFloat()));
    List<INDArray> activations1 = net.feedForward(in1);
    List<INDArray> activations2 = net.feedForward(in2);
    for (int i = 0; i < 3; i++) {
        System.out.println("-----\n" + i);
        System.out.println(Arrays.toString(activations1.get(i).dup().data().asDouble()));
        System.out.println(Arrays.toString(activations2.get(i).dup().data().asDouble()));
        System.out.println(activations1.get(i));
        System.out.println(activations2.get(i));
    }
    //Expect first 4 time steps to be indentical...
    for (int i = 0; i < 4; i++) {
        double d1 = out1.getDouble(i);
        double d2 = out2.getDouble(i);
        assertEquals(d1, d2, 0.0);
    }
}

Example 9 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class GravesLSTMTest method testGateActivationFnsSanityCheck.

@Test
public void testGateActivationFnsSanityCheck() {
    for (String gateAfn : new String[] { "sigmoid", "hardsigmoid" }) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2).activation(Activation.TANH).build()).build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesLSTM) net.getLayer(0).conf().getLayer()).getGateActivationFn().toString());
        INDArray in = Nd4j.rand(new int[] { 3, 2, 5 });
        INDArray labels = Nd4j.rand(new int[] { 3, 2, 5 });
        net.fit(in, labels);
    }
}

Example 10 with MultiLayerNetwork

use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.

the class BatchNormalizationTest method checkMeanVarianceEstimateCNN.

@Test
public void checkMeanVarianceEstimateCNN() throws Exception {
    Nd4j.getRandom().setSeed(12345);
    //Check that the internal global mean/variance estimate is approximately correct
    //First, Mnist data as 2d input (NOT taking into account convolution property)
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1).updater(Updater.RMSPROP).seed(12345).list().layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).nOut(10).build()).backprop(true).pretrain(false).setInputType(InputType.convolutional(5, 5, 3)).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    int minibatch = 32;
    List<DataSet> list = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        list.add(new DataSet(Nd4j.rand(new int[] { minibatch, 3, 5, 5 }), Nd4j.rand(minibatch, 10)));
    }
    DataSetIterator iter = new ListDataSetIterator(list);
    INDArray expMean = Nd4j.valueArrayOf(new int[] { 1, 3 }, 0.5);
    //Expected variance of U(0,1) distribution: 1/12 * (1-0)^2 = 0.0833
    INDArray expVar = Nd4j.valueArrayOf(new int[] { 1, 3 }, 1 / 12.0);
    for (int i = 0; i < 10; i++) {
        iter.reset();
        net.fit(iter);
    }
    INDArray estMean = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_MEAN);
    INDArray estVar = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_VAR);
    float[] fMeanExp = expMean.data().asFloat();
    float[] fMeanAct = estMean.data().asFloat();
    float[] fVarExp = expVar.data().asFloat();
    float[] fVarAct = estVar.data().asFloat();
    //        System.out.println("Mean vs. estimated mean:");
    //        System.out.println(Arrays.toString(fMeanExp));
    //        System.out.println(Arrays.toString(fMeanAct));
    //
    //        System.out.println("Var vs. estimated var:");
    //        System.out.println(Arrays.toString(fVarExp));
    //        System.out.println(Arrays.toString(fVarAct));
    assertArrayEquals(fMeanExp, fMeanAct, 0.01f);
    assertArrayEquals(fVarExp, fVarAct, 0.01f);
}