Examples with GaussianReconstructionDistribution org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution used on opensource projects

Example 1 with GaussianReconstructionDistribution

use of org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution in project deeplearning4j by deeplearning4j.

the class TestReconstructionDistributions method testGaussianLogProb.

@Test
public void testGaussianLogProb() {
    Nd4j.getRandom().setSeed(12345);
    int inputSize = 4;
    int[] mbs = new int[] { 1, 2, 5 };
    for (boolean average : new boolean[] { true, false }) {
        for (int minibatch : mbs) {
            INDArray x = Nd4j.rand(minibatch, inputSize);
            INDArray mean = Nd4j.randn(minibatch, inputSize);
            INDArray logStdevSquared = Nd4j.rand(minibatch, inputSize).subi(0.5);
            INDArray distributionParams = Nd4j.createUninitialized(new int[] { minibatch, 2 * inputSize });
            distributionParams.get(NDArrayIndex.all(), NDArrayIndex.interval(0, inputSize)).assign(mean);
            distributionParams.get(NDArrayIndex.all(), NDArrayIndex.interval(inputSize, 2 * inputSize)).assign(logStdevSquared);
            ReconstructionDistribution dist = new GaussianReconstructionDistribution("identity");
            double negLogProb = dist.negLogProbability(x, distributionParams, average);
            INDArray exampleNegLogProb = dist.exampleNegLogProbability(x, distributionParams);
            assertArrayEquals(new int[] { minibatch, 1 }, exampleNegLogProb.shape());
            //Calculate the same thing, but using Apache Commons math
            double logProbSum = 0.0;
            for (int i = 0; i < minibatch; i++) {
                double exampleSum = 0.0;
                for (int j = 0; j < inputSize; j++) {
                    double mu = mean.getDouble(i, j);
                    double logSigma2 = logStdevSquared.getDouble(i, j);
                    double sigma = Math.sqrt(Math.exp(logSigma2));
                    NormalDistribution nd = new NormalDistribution(mu, sigma);
                    double xVal = x.getDouble(i, j);
                    double thisLogProb = nd.logDensity(xVal);
                    logProbSum += thisLogProb;
                    exampleSum += thisLogProb;
                }
                assertEquals(-exampleNegLogProb.getDouble(i), exampleSum, 1e-6);
            }
            double expNegLogProb;
            if (average) {
                expNegLogProb = -logProbSum / minibatch;
            } else {
                expNegLogProb = -logProbSum;
            }
            //                System.out.println(expLogProb + "\t" + logProb + "\t" + (logProb / expLogProb));
            assertEquals(expNegLogProb, negLogProb, 1e-6);
            //Also: check random sampling...
            int count = minibatch * inputSize;
            INDArray arr = Nd4j.linspace(-3, 3, count).reshape(minibatch, inputSize);
            INDArray sampleMean = dist.generateAtMean(arr);
            INDArray sampleRandom = dist.generateRandom(arr);
        }
    }
}

Example 2 with GaussianReconstructionDistribution

use of org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution in project deeplearning4j by deeplearning4j.

the class TestReconstructionDistributions method gradientCheckReconstructionDistributions.

@Test
public void gradientCheckReconstructionDistributions() {
    double eps = 1e-6;
    double maxRelError = 1e-6;
    double minAbsoluteError = 1e-9;
    Nd4j.getRandom().setSeed(12345);
    int inputSize = 4;
    int[] mbs = new int[] { 1, 3 };
    Random r = new Random(12345);
    ReconstructionDistribution[] distributions = new ReconstructionDistribution[] { new GaussianReconstructionDistribution("identity"), new GaussianReconstructionDistribution("tanh"), new BernoulliReconstructionDistribution("sigmoid"), new ExponentialReconstructionDistribution("identity"), new ExponentialReconstructionDistribution("tanh") };
    List<String> passes = new ArrayList<>();
    List<String> failures = new ArrayList<>();
    for (ReconstructionDistribution rd : distributions) {
        for (int minibatch : mbs) {
            INDArray x;
            INDArray distributionParams;
            if (rd instanceof GaussianReconstructionDistribution) {
                distributionParams = Nd4j.rand(minibatch, inputSize * 2).muli(2).subi(1);
                x = Nd4j.rand(minibatch, inputSize);
            } else if (rd instanceof BernoulliReconstructionDistribution) {
                distributionParams = Nd4j.rand(minibatch, inputSize).muli(2).subi(1);
                x = Nd4j.zeros(minibatch, inputSize);
                for (int i = 0; i < minibatch; i++) {
                    for (int j = 0; j < inputSize; j++) {
                        x.putScalar(i, j, r.nextInt(2));
                    }
                }
            } else if (rd instanceof ExponentialReconstructionDistribution) {
                distributionParams = Nd4j.rand(minibatch, inputSize).muli(2).subi(1);
                x = Nd4j.rand(minibatch, inputSize);
            } else {
                throw new RuntimeException();
            }
            INDArray gradient = rd.gradient(x, distributionParams);
            String testName = "minibatch = " + minibatch + ", size = " + inputSize + ", Distribution = " + rd;
            System.out.println("\n\n***** Starting test: " + testName + "*****");
            int totalFailureCount = 0;
            for (int i = 0; i < distributionParams.size(1); i++) {
                for (int j = 0; j < distributionParams.size(0); j++) {
                    double initial = distributionParams.getDouble(j, i);
                    distributionParams.putScalar(j, i, initial + eps);
                    double scorePlus = rd.negLogProbability(x, distributionParams, false);
                    distributionParams.putScalar(j, i, initial - eps);
                    double scoreMinus = rd.negLogProbability(x, distributionParams, false);
                    distributionParams.putScalar(j, i, initial);
                    double numericalGrad = (scorePlus - scoreMinus) / (2.0 * eps);
                    double backpropGrad = gradient.getDouble(j, i);
                    double relError = Math.abs(numericalGrad - backpropGrad) / (Math.abs(numericalGrad) + Math.abs(backpropGrad));
                    double absError = Math.abs(backpropGrad - numericalGrad);
                    if (relError > maxRelError || Double.isNaN(relError)) {
                        if (absError < minAbsoluteError) {
                            log.info("Input (" + j + "," + i + ") passed: grad= " + backpropGrad + ", numericalGrad= " + numericalGrad + ", relError= " + relError + "; absolute error = " + absError + " < minAbsoluteError = " + minAbsoluteError);
                        } else {
                            log.info("Input (" + j + "," + i + ") FAILED: grad= " + backpropGrad + ", numericalGrad= " + numericalGrad + ", relError= " + relError + ", scorePlus=" + scorePlus + ", scoreMinus= " + scoreMinus);
                            totalFailureCount++;
                        }
                    } else {
                        log.info("Input (" + j + "," + i + ") passed: grad= " + backpropGrad + ", numericalGrad= " + numericalGrad + ", relError= " + relError);
                    }
                }
            }
            if (totalFailureCount > 0) {
                failures.add(testName);
            } else {
                passes.add(testName);
            }
        }
    }
    System.out.println("\n\n\n +++++ Test Passes +++++");
    for (String s : passes) {
        System.out.println(s);
    }
    System.out.println("\n\n\n +++++ Test Faliures +++++");
    for (String s : failures) {
        System.out.println(s);
    }
    assertEquals(0, failures.size());
}

Example 3 with GaussianReconstructionDistribution

use of org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testVaePretrainSimple.

@Test
public void testVaePretrainSimple() {
    //Simple sanity check on pretraining
    int nIn = 8;
    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.RMSPROP).weightInit(WeightInit.XAVIER).list().layer(0, new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12).decoderLayerSizes(13).reconstructionDistribution(new GaussianReconstructionDistribution("identity")).build()).pretrain(true).backprop(false).build();
    //Do training on Spark with one executor, for 3 separate minibatches
    int rddDataSetNumExamples = 10;
    int totalAveragings = 5;
    int averagingFrequency = 3;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples).averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples).saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);
    SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf.clone(), tm);
    List<DataSet> trainData = new ArrayList<>();
    int nDataSets = numExecutors() * totalAveragings * averagingFrequency;
    for (int i = 0; i < nDataSets; i++) {
        trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null));
    }
    JavaRDD<DataSet> data = sc.parallelize(trainData);
    sparkNet.fit(data);
}

Example 4 with GaussianReconstructionDistribution

use of org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution in project deeplearning4j by deeplearning4j.

the class TestSparkMultiLayerParameterAveraging method testVaePretrainSimpleCG.

@Test
public void testVaePretrainSimpleCG() {
    //Simple sanity check on pretraining
    int nIn = 8;
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.RMSPROP).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("0", new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12).decoderLayerSizes(13).reconstructionDistribution(new GaussianReconstructionDistribution("identity")).build(), "in").setOutputs("0").pretrain(true).backprop(false).build();
    //Do training on Spark with one executor, for 3 separate minibatches
    int rddDataSetNumExamples = 10;
    int totalAveragings = 5;
    int averagingFrequency = 3;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples).averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples).saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);
    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), tm);
    List<DataSet> trainData = new ArrayList<>();
    int nDataSets = numExecutors() * totalAveragings * averagingFrequency;
    for (int i = 0; i < nDataSets; i++) {
        trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null));
    }
    JavaRDD<DataSet> data = sc.parallelize(trainData);
    sparkNet.fit(data);
}

Example 5 with GaussianReconstructionDistribution

use of org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution in project deeplearning4j by deeplearning4j.

the class TestMiscFunctions method testVaeReconstructionProbabilityWithKey.

@Test
public void testVaeReconstructionProbabilityWithKey() {
    //Simple test. We can't do a direct comparison, as the reconstruction probabilities are stochastic
    // due to sampling
    int nIn = 10;
    MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().list().layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder().reconstructionDistribution(new GaussianReconstructionDistribution(Activation.IDENTITY)).nIn(nIn).nOut(5).encoderLayerSizes(12).decoderLayerSizes(13).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();
    List<Tuple2<Integer, INDArray>> toScore = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        INDArray arr = Nd4j.rand(1, nIn);
        toScore.add(new Tuple2<Integer, INDArray>(i, arr));
    }
    JavaPairRDD<Integer, INDArray> rdd = sc.parallelizePairs(toScore);
    JavaPairRDD<Integer, Double> reconstr = rdd.mapPartitionsToPair(new VaeReconstructionProbWithKeyFunction<Integer>(sc.broadcast(net.params()), sc.broadcast(mlc.toJson()), true, 16, 128));
    Map<Integer, Double> l = reconstr.collectAsMap();
    assertEquals(100, l.size());
    for (int i = 0; i < 100; i++) {
        assertTrue(l.containsKey(i));
        //log probability: should be negative
        assertTrue(l.get(i) < 0.0);
    }
}