Examples with ActivationTanH org.nd4j.linalg.activations.impl.ActivationTanH used on opensource projects

Example 1 with ActivationTanH

use of org.nd4j.linalg.activations.impl.ActivationTanH in project deeplearning4j by deeplearning4j.

the class TestVAE method testReconstructionDistributionsSimple.

@Test
public void testReconstructionDistributionsSimple() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new GaussianReconstructionDistribution(Activation.IDENTITY), new GaussianReconstructionDistribution(Activation.TANH), new BernoulliReconstructionDistribution(Activation.SIGMOID), new CompositeReconstructionDistribution.Builder().addDistribution(2, new GaussianReconstructionDistribution(Activation.IDENTITY)).addDistribution(2, new BernoulliReconstructionDistribution()).addDistribution(2, new GaussianReconstructionDistribution(Activation.TANH)).build() };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data;
            switch(i) {
                //Gaussian + identity
                case 0:
                case //Gaussian + tanh
                1:
                    data = Nd4j.rand(minibatch, inOutSize);
                    break;
                case //Bernoulli
                2:
                    data = Nd4j.create(minibatch, inOutSize);
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data, 0.5), Nd4j.getRandom());
                    break;
                case //Composite
                3:
                    data = Nd4j.create(minibatch, inOutSize);
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 2)).assign(Nd4j.rand(minibatch, 2));
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data.get(NDArrayIndex.all(), NDArrayIndex.interval(2, 4)), 0.5), Nd4j.getRandom());
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(4, 6)).assign(Nd4j.rand(minibatch, 2));
                    break;
                default:
                    throw new RuntimeException();
            }
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(new ActivationTanH()).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            mln.fit(data);
            org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) mln.getLayer(0);
            assertFalse(layer.hasLossFunction());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionProb = layer.reconstructionProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionProb.shape());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionLogProb = layer.reconstructionLogProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionLogProb.shape());
            //                System.out.println(reconstructionDistributions[i]);
            for (int j = 0; j < minibatch; j++) {
                double p = reconstructionProb.getDouble(j);
                double logp = reconstructionLogProb.getDouble(j);
                assertTrue(p >= 0.0 && p <= 1.0);
                assertTrue(logp <= 0.0);
                double pFromLogP = Math.exp(logp);
                assertEquals(p, pFromLogP, 1e-6);
            }
        }
    }
}

Example 2 with ActivationTanH

use of org.nd4j.linalg.activations.impl.ActivationTanH in project deeplearning4j by deeplearning4j.

the class TestVAE method testJsonYaml.

@Test
public void testJsonYaml() {
    MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new GaussianReconstructionDistribution("identity")).nIn(3).nOut(4).encoderLayerSizes(5).decoderLayerSizes(6).build()).layer(1, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new GaussianReconstructionDistribution("tanh")).nIn(7).nOut(8).encoderLayerSizes(9).decoderLayerSizes(10).build()).layer(2, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new BernoulliReconstructionDistribution()).nIn(11).nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build()).layer(3, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new ExponentialReconstructionDistribution("tanh")).nIn(11).nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build()).layer(4, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().lossFunction(new ActivationTanH(), LossFunctions.LossFunction.MSE).nIn(11).nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build()).layer(5, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new CompositeReconstructionDistribution.Builder().addDistribution(5, new GaussianReconstructionDistribution()).addDistribution(5, new GaussianReconstructionDistribution("tanh")).addDistribution(5, new BernoulliReconstructionDistribution()).build()).nIn(15).nOut(16).encoderLayerSizes(17).decoderLayerSizes(18).build()).layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(18).nOut(19).activation(new ActivationTanH()).build()).pretrain(true).backprop(true).build();
    String asJson = config.toJson();
    String asYaml = config.toYaml();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson);
    MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(asYaml);
    assertEquals(config, fromJson);
    assertEquals(config, fromYaml);
}

Example 3 with ActivationTanH

use of org.nd4j.linalg.activations.impl.ActivationTanH in project deeplearning4j by deeplearning4j.

the class VaeGradientCheckTests method testVaePretrainReconstructionDistributions.

@Test
public void testVaePretrainReconstructionDistributions() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new GaussianReconstructionDistribution(Activation.IDENTITY), new GaussianReconstructionDistribution(Activation.TANH), new BernoulliReconstructionDistribution(Activation.SIGMOID), new CompositeReconstructionDistribution.Builder().addDistribution(2, new GaussianReconstructionDistribution(Activation.IDENTITY)).addDistribution(2, new BernoulliReconstructionDistribution()).addDistribution(2, new GaussianReconstructionDistribution(Activation.TANH)).build(), new ExponentialReconstructionDistribution("identity"), new ExponentialReconstructionDistribution("tanh"), new LossFunctionWrapper(new ActivationTanH(), new LossMSE()), new LossFunctionWrapper(new ActivationIdentity(), new LossMAE()) };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data;
            switch(i) {
                //Gaussian + identity
                case 0:
                case //Gaussian + tanh
                1:
                    data = Nd4j.rand(minibatch, inOutSize);
                    break;
                case //Bernoulli
                2:
                    data = Nd4j.create(minibatch, inOutSize);
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data, 0.5), Nd4j.getRandom());
                    break;
                case //Composite
                3:
                    data = Nd4j.create(minibatch, inOutSize);
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 2)).assign(Nd4j.rand(minibatch, 2));
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data.get(NDArrayIndex.all(), NDArrayIndex.interval(2, 4)), 0.5), Nd4j.getRandom());
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(4, 6)).assign(Nd4j.rand(minibatch, 2));
                    break;
                case 4:
                case 5:
                    data = Nd4j.rand(minibatch, inOutSize);
                    break;
                case 6:
                case 7:
                    data = Nd4j.randn(minibatch, inOutSize);
                    break;
                default:
                    throw new RuntimeException();
            }
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(Activation.TANH).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            org.deeplearning4j.nn.api.Layer layer = mln.getLayer(0);
            String msg = "testVaePretrainReconstructionDistributions() - " + reconstructionDistributions[i];
            if (PRINT_RESULTS) {
                System.out.println(msg);
                for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
            }
            boolean gradOK = GradientCheckUtil.checkGradientsPretrainLayer(layer, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, data, 12345);
            assertTrue(msg, gradOK);
        }
    }
}

Example 4 with ActivationTanH

use of org.nd4j.linalg.activations.impl.ActivationTanH in project deeplearning4j by deeplearning4j.

the class TestVAE method testPretrainParamsDuringBackprop.

@Test
public void testPretrainParamsDuringBackprop() {
    //Idea: pretrain-specific parameters shouldn't change during backprop
    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().nIn(10).nOut(5).encoderLayerSizes(12, 13).decoderLayerSizes(14, 15).build()).layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).activation(new ActivationTanH()).build()).pretrain(true).backprop(true).build();
    NeuralNetConfiguration c = mlc.getConf(0);
    org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder vae = (org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder) c.getLayer();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();
    net.initGradientsView();
    org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) net.getLayer(0);
    INDArray input = Nd4j.rand(3, 10);
    net.pretrain(input);
    //Get a snapshot of the pretrain params after fitting:
    Map<String, INDArray> layerParams = layer.paramTable();
    Map<String, INDArray> pretrainParamsBefore = new HashMap<>();
    for (String s : layerParams.keySet()) {
        if (layer.isPretrainParam(s)) {
            pretrainParamsBefore.put(s, layerParams.get(s).dup());
        }
    }
    INDArray features = Nd4j.rand(3, 10);
    INDArray labels = Nd4j.rand(3, 6);
    net.getLayerWiseConfigurations().setPretrain(false);
    for (int i = 0; i < 3; i++) {
        net.fit(features, labels);
    }
    Map<String, INDArray> layerParamsAfter = layer.paramTable();
    for (String s : pretrainParamsBefore.keySet()) {
        INDArray before = pretrainParamsBefore.get(s);
        INDArray after = layerParamsAfter.get(s);
        assertEquals(before, after);
    }
}

Example 5 with ActivationTanH

use of org.nd4j.linalg.activations.impl.ActivationTanH in project deeplearning4j by deeplearning4j.

the class TestVAE method testReconstructionErrorSimple.

@Test
public void testReconstructionErrorSimple() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new LossFunctionWrapper(Activation.TANH, new LossMSE()), new LossFunctionWrapper(Activation.IDENTITY, new LossMAE()), new CompositeReconstructionDistribution.Builder().addDistribution(3, new LossFunctionWrapper(Activation.TANH, new LossMSE())).addDistribution(3, new LossFunctionWrapper(Activation.IDENTITY, new LossMAE())).build() };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data = Nd4j.rand(minibatch, inOutSize).muli(2).subi(1);
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(new ActivationTanH()).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            mln.fit(data);
            org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) mln.getLayer(0);
            assertTrue(layer.hasLossFunction());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionError = layer.reconstructionError(data);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionError.shape());
            for (int j = 0; j < minibatch; j++) {
                double re = reconstructionError.getDouble(j);
                assertTrue(re >= 0.0);
            }
        }
    }
}