Examples with MultiLayerConfiguration org.deeplearning4j.nn.conf.MultiLayerConfiguration used on opensource projects

Example 71 with MultiLayerConfiguration

use of org.deeplearning4j.nn.conf.MultiLayerConfiguration in project deeplearning4j by deeplearning4j.

the class EmbeddingLayerTest method testEmbeddingBackwardPass.

@Test
public void testEmbeddingBackwardPass() {
    //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation
    // input with a DenseLayer
    int nClassesIn = 10;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).pretrain(false).backprop(true).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();
    net2.setParams(net.params().dup());
    int batchSize = 3;
    INDArray inEmbedding = Nd4j.create(batchSize, 1);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn);
    INDArray outLabels = Nd4j.create(batchSize, 4);
    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
        inOneHot.putScalar(new int[] { i, classIdx }, 1.0);
        int labelIdx = r.nextInt(4);
        outLabels.putScalar(new int[] { i, labelIdx }, 1.0);
    }
    net.setInput(inEmbedding);
    net2.setInput(inOneHot);
    net.setLabels(outLabels);
    net2.setLabels(outLabels);
    net.computeGradientAndScore();
    net2.computeGradientAndScore();
    System.out.println(net.score() + "\t" + net2.score());
    assertEquals(net2.score(), net.score(), 1e-6);
    Map<String, INDArray> gradient = net.gradient().gradientForVariable();
    Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
    assertEquals(gradient.size(), gradient2.size());
    for (String s : gradient.keySet()) {
        assertEquals(gradient2.get(s), gradient.get(s));
    }
}

Example 72 with MultiLayerConfiguration

use of org.deeplearning4j.nn.conf.MultiLayerConfiguration in project deeplearning4j by deeplearning4j.

the class EmbeddingLayerTest method testEmbeddingForwardPass.

@Test
public void testEmbeddingForwardPass() {
    //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation
    // input with a DenseLayer
    int nClassesIn = 10;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).build()).pretrain(false).backprop(true).build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).build()).pretrain(false).backprop(true).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();
    net2.setParams(net.params().dup());
    int batchSize = 3;
    INDArray inEmbedding = Nd4j.create(batchSize, 1);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn);
    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
        inOneHot.putScalar(new int[] { i, classIdx }, 1.0);
    }
    List<INDArray> activationsEmbedding = net.feedForward(inEmbedding, false);
    List<INDArray> activationsDense = net2.feedForward(inOneHot, false);
    for (int i = 1; i < 3; i++) {
        INDArray actE = activationsEmbedding.get(i);
        INDArray actD = activationsDense.get(i);
        assertEquals(actE, actD);
    }
}

Example 73 with MultiLayerConfiguration

use of org.deeplearning4j.nn.conf.MultiLayerConfiguration in project deeplearning4j by deeplearning4j.

the class EmbeddingLayerTest method testEmbeddingLayerRNN.

@Test
public void testEmbeddingLayerRNN() {
    int nClassesIn = 10;
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()).layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(1, new FeedForwardToRnnPreProcessor()).pretrain(false).backprop(true).build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).build()).layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()).layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(1, new FeedForwardToRnnPreProcessor()).pretrain(false).backprop(true).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();
    net2.setParams(net.params().dup());
    int batchSize = 3;
    int timeSeriesLength = 8;
    INDArray inEmbedding = Nd4j.create(batchSize, 1, timeSeriesLength);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, timeSeriesLength);
    INDArray outLabels = Nd4j.create(batchSize, 4, timeSeriesLength);
    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        for (int j = 0; j < timeSeriesLength; j++) {
            int classIdx = r.nextInt(nClassesIn);
            inEmbedding.putScalar(new int[] { i, 0, j }, classIdx);
            inOneHot.putScalar(new int[] { i, classIdx, j }, 1.0);
            int labelIdx = r.nextInt(4);
            outLabels.putScalar(new int[] { i, labelIdx, j }, 1.0);
        }
    }
    net.setInput(inEmbedding);
    net2.setInput(inOneHot);
    net.setLabels(outLabels);
    net2.setLabels(outLabels);
    net.computeGradientAndScore();
    net2.computeGradientAndScore();
    System.out.println(net.score() + "\t" + net2.score());
    assertEquals(net2.score(), net.score(), 1e-6);
    Map<String, INDArray> gradient = net.gradient().gradientForVariable();
    Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
    assertEquals(gradient.size(), gradient2.size());
    for (String s : gradient.keySet()) {
        assertEquals(gradient2.get(s), gradient.get(s));
    }
}

Example 74 with MultiLayerConfiguration

use of org.deeplearning4j.nn.conf.MultiLayerConfiguration in project deeplearning4j by deeplearning4j.

the class RBMTests method getMultiLayerRBMNet.

private static MultiLayerNetwork getMultiLayerRBMNet(boolean backprop, boolean pretrain, INDArray input, int nOut1, int nOut2, int nOut3, WeightInit weightInit) {
    MultiLayerConfiguration rbm = new NeuralNetConfiguration.Builder().seed(0xDEADBEEF).iterations(1000).biasInit(0).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.NONE).epsilon(1).weightInit(weightInit).list(new org.deeplearning4j.nn.conf.layers.RBM.Builder().lossFunction(LossFunctions.LossFunction.KL_DIVERGENCE).nOut(nOut1).build(), new org.deeplearning4j.nn.conf.layers.RBM.Builder().lossFunction(LossFunctions.LossFunction.KL_DIVERGENCE).nOut(nOut2).build(), new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.RELU).nOut(nOut3).build()).pretrain(pretrain).backprop(backprop).setInputType(InputType.feedForward(input.columns())).build();
    MultiLayerNetwork network = new MultiLayerNetwork(rbm);
    network.init();
    return network;
}

Example 75 with MultiLayerConfiguration

use of org.deeplearning4j.nn.conf.MultiLayerConfiguration in project deeplearning4j by deeplearning4j.

the class TestCustomLayers method testCustomOutputLayerMLN.

@Test
public void testCustomOutputLayerMLN() {
    //First: Ensure that the CustomOutputLayer class is registered
    ObjectMapper mapper = NeuralNetConfiguration.mapper();
    AnnotatedClass ac = AnnotatedClass.construct(Layer.class, mapper.getSerializationConfig().getAnnotationIntrospector(), null);
    Collection<NamedType> types = mapper.getSubtypeResolver().collectAndResolveSubtypes(ac, mapper.getSerializationConfig(), mapper.getSerializationConfig().getAnnotationIntrospector());
    Set<Class<?>> registeredSubtypes = new HashSet<>();
    boolean found = false;
    for (NamedType nt : types) {
        System.out.println(nt);
        //            registeredSubtypes.add(nt.getType());
        if (nt.getType() == CustomOutputLayer.class)
            found = true;
    }
    assertTrue("CustomOutputLayer: not registered with NeuralNetConfiguration mapper", found);
    //Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works...
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).learningRate(0.1).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new CustomOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10).nOut(10).build()).pretrain(false).backprop(true).build();
    String json = conf.toJson();
    String yaml = conf.toYaml();
    System.out.println(json);
    MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, confFromJson);
    MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml);
    assertEquals(conf, confFromYaml);
    //Third: check initialization
    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    assertTrue(net.getLayer(1) instanceof CustomOutputLayerImpl);
    //Fourth: compare to an equivalent standard output layer (should be identical)
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).learningRate(0.1).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10).nOut(10).build()).pretrain(false).backprop(true).build();
    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();
    assertEquals(net2.params(), net.params());
    INDArray testFeatures = Nd4j.rand(1, 10);
    INDArray testLabels = Nd4j.zeros(1, 10);
    testLabels.putScalar(0, 3, 1.0);
    DataSet ds = new DataSet(testFeatures, testLabels);
    assertEquals(net2.output(testFeatures), net.output(testFeatures));
    assertEquals(net2.score(ds), net.score(ds), 1e-6);
}