Examples with NeuralNetConfiguration org.deeplearning4j.nn.conf.NeuralNetConfiguration used on opensource projects

Example 21 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestDecayPolicies method testLearningRateStepDecaySingleLayer.

@Test
public void testLearningRateStepDecaySingleLayer() {
    int iterations = 2;
    double lr = 1e-2;
    double decayRate = 2;
    double steps = 3;
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).learningRateDecayPolicy(LearningRatePolicy.Step).lrPolicyDecayRate(decayRate).lrPolicySteps(steps).iterations(iterations).layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).updater(org.deeplearning4j.nn.conf.Updater.SGD).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);
    Updater updater = UpdaterCreator.getUpdater(layer);
    Gradient gradientActual = new DefaultGradient();
    gradientActual.setGradientFor(DefaultParamInitializer.WEIGHT_KEY, weightGradient);
    gradientActual.setGradientFor(DefaultParamInitializer.BIAS_KEY, biasGradient);
    for (int i = 0; i < iterations; i++) {
        updater.update(layer, gradientActual, i, 1);
        double expectedLr = calcStepDecay(lr, decayRate, i, steps);
        assertEquals(expectedLr, layer.conf().getLearningRateByParam("W"), 1e-4);
        assertEquals(expectedLr, layer.conf().getLearningRateByParam("b"), 1e-4);
    }
}

Example 22 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class GravesLSTMTest method testGravesLSTMForwardPassHelper.

@Test
public void testGravesLSTMForwardPassHelper() throws Exception {
    //GravesLSTM.activateHelper() has different behaviour (due to optimizations) when forBackprop==true vs false
    //But should otherwise provide identical activations
    Nd4j.getRandom().setSeed(12345);
    int nIn = 10;
    int layerSize = 15;
    int miniBatchSize = 4;
    int timeSeriesLength = 7;
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(layerSize).weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true);
    INDArray input = Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength });
    lstm.setInput(input);
    Method actHelper = GravesLSTM.class.getDeclaredMethod("activateHelper", boolean.class, INDArray.class, INDArray.class, boolean.class);
    actHelper.setAccessible(true);
    //Call activateHelper with both forBackprop == true, and forBackprop == false and compare
    Class<?> innerClass = Class.forName("org.deeplearning4j.nn.layers.recurrent.FwdPassReturn");
    //GravesLSTM.FwdPassReturn object; want fwdPassOutput INDArray
    Object oFalse = actHelper.invoke(lstm, false, null, null, false);
    //want fwdPassOutputAsArrays object
    Object oTrue = actHelper.invoke(lstm, false, null, null, true);
    Field fwdPassOutput = innerClass.getDeclaredField("fwdPassOutput");
    fwdPassOutput.setAccessible(true);
    Field fwdPassOutputAsArrays = innerClass.getDeclaredField("fwdPassOutputAsArrays");
    fwdPassOutputAsArrays.setAccessible(true);
    INDArray fwdPassFalse = (INDArray) fwdPassOutput.get(oFalse);
    INDArray[] fwdPassTrue = (INDArray[]) fwdPassOutputAsArrays.get(oTrue);
    for (int i = 0; i < timeSeriesLength; i++) {
        INDArray sliceFalse = fwdPassFalse.tensorAlongDimension(i, 1, 0);
        INDArray sliceTrue = fwdPassTrue[i];
        assertTrue(sliceFalse.equals(sliceTrue));
    }
}

Example 23 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestVAE method testForwardPass.

@Test
public void testForwardPass() {
    int[][] encLayerSizes = new int[][] { { 12 }, { 12, 13 }, { 12, 13, 14 } };
    for (int i = 0; i < encLayerSizes.length; i++) {
        MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().list().layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().nIn(10).nOut(5).encoderLayerSizes(encLayerSizes[i]).decoderLayerSizes(13).build()).build();
        NeuralNetConfiguration c = mlc.getConf(0);
        org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder vae = (org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder) c.getLayer();
        MultiLayerNetwork net = new MultiLayerNetwork(mlc);
        net.init();
        INDArray in = Nd4j.rand(1, 10);
        //        net.output(in);
        List<INDArray> out = net.feedForward(in);
        assertArrayEquals(new int[] { 1, 10 }, out.get(0).shape());
        assertArrayEquals(new int[] { 1, 5 }, out.get(1).shape());
    }
}

Example 24 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestOptimizers method testRastriginFnMultipleStepsHelper.

private static void testRastriginFnMultipleStepsHelper(OptimizationAlgorithm oa, int nOptIter, int maxNumLineSearchIter) {
    double[] scores = new double[nOptIter + 1];
    for (int i = 0; i <= nOptIter; i++) {
        NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().maxNumLineSearchIterations(maxNumLineSearchIter).iterations(i).miniBatch(false).learningRate(1e-2).layer(new DenseLayer.Builder().nIn(1).nOut(1).updater(Updater.ADAGRAD).build()).build();
        //Normally done by ParamInitializers, but obviously that isn't done here
        conf.addVariable("W");
        Model m = new RastriginFunctionModel(100, conf);
        int nParams = m.numParams();
        if (i == 0) {
            m.computeGradientAndScore();
            //Before optimization
            scores[0] = m.score();
        } else {
            ConvexOptimizer opt = getOptimizer(oa, conf, m);
            opt.getUpdater().setStateViewArray((Layer) m, Nd4j.createUninitialized(new int[] { 1, nParams }, 'c'), true);
            opt.optimize();
            m.computeGradientAndScore();
            scores[i] = m.score();
            assertTrue(!Double.isNaN(scores[i]) && !Double.isInfinite(scores[i]));
        }
    }
    if (PRINT_OPT_RESULTS) {
        System.out.println("Rastrigin: Multiple optimization iterations (" + nOptIter + " opt. iter.) score vs iteration, maxNumLineSearchIter=" + maxNumLineSearchIter + ": " + oa);
        System.out.println(Arrays.toString(scores));
    }
    for (int i = 1; i < scores.length; i++) {
        if (i == 1) {
            //Require at least one step of improvement
            assertTrue(scores[i] <= scores[i - 1]);
        } else {
            assertTrue(scores[i] <= scores[i - 1]);
        }
    }
}

Example 25 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestOptimizers method testSphereFnOptHelper.

public void testSphereFnOptHelper(OptimizationAlgorithm oa, int numLineSearchIter, int nDimensions) {
    if (PRINT_OPT_RESULTS)
        System.out.println("---------\n Alg= " + oa + ", nIter= " + numLineSearchIter + ", nDimensions= " + nDimensions);
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().maxNumLineSearchIterations(numLineSearchIter).iterations(100).learningRate(1e-2).layer(new RBM.Builder().nIn(1).nOut(1).updater(Updater.SGD).build()).build();
    //Normally done by ParamInitializers, but obviously that isn't done here
    conf.addVariable("W");
    Random rng = new DefaultRandom(12345L);
    org.nd4j.linalg.api.rng.distribution.Distribution dist = new org.nd4j.linalg.api.rng.distribution.impl.UniformDistribution(rng, -10, 10);
    Model m = new SphereFunctionModel(nDimensions, dist, conf);
    m.computeGradientAndScore();
    double scoreBefore = m.score();
    assertTrue(!Double.isNaN(scoreBefore) && !Double.isInfinite(scoreBefore));
    if (PRINT_OPT_RESULTS) {
        System.out.println("Before:");
        System.out.println(scoreBefore);
        System.out.println(m.params());
    }
    ConvexOptimizer opt = getOptimizer(oa, conf, m);
    opt.setupSearchState(m.gradientAndScore());
    opt.optimize();
    m.computeGradientAndScore();
    double scoreAfter = m.score();
    assertTrue(!Double.isNaN(scoreAfter) && !Double.isInfinite(scoreAfter));
    if (PRINT_OPT_RESULTS) {
        System.out.println("After:");
        System.out.println(scoreAfter);
        System.out.println(m.params());
    }
    //Expected behaviour after optimization:
    //(a) score is better (lower) after optimization.
    //(b) Parameters are closer to minimum after optimization (TODO)
    assertTrue("Score did not improve after optimization (b= " + scoreBefore + " ,a= " + scoreAfter + ")", scoreAfter < scoreBefore);
}