Examples with BernoulliDistribution org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution used on opensource projects

Example 1 with BernoulliDistribution

use of org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution in project deeplearning4j by deeplearning4j.

the class TestVAE method testReconstructionDistributionsSimple.

@Test
public void testReconstructionDistributionsSimple() {
    int inOutSize = 6;
    ReconstructionDistribution[] reconstructionDistributions = new ReconstructionDistribution[] { new GaussianReconstructionDistribution(Activation.IDENTITY), new GaussianReconstructionDistribution(Activation.TANH), new BernoulliReconstructionDistribution(Activation.SIGMOID), new CompositeReconstructionDistribution.Builder().addDistribution(2, new GaussianReconstructionDistribution(Activation.IDENTITY)).addDistribution(2, new BernoulliReconstructionDistribution()).addDistribution(2, new GaussianReconstructionDistribution(Activation.TANH)).build() };
    Nd4j.getRandom().setSeed(12345);
    for (int minibatch : new int[] { 1, 5 }) {
        for (int i = 0; i < reconstructionDistributions.length; i++) {
            INDArray data;
            switch(i) {
                //Gaussian + identity
                case 0:
                case //Gaussian + tanh
                1:
                    data = Nd4j.rand(minibatch, inOutSize);
                    break;
                case //Bernoulli
                2:
                    data = Nd4j.create(minibatch, inOutSize);
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data, 0.5), Nd4j.getRandom());
                    break;
                case //Composite
                3:
                    data = Nd4j.create(minibatch, inOutSize);
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 2)).assign(Nd4j.rand(minibatch, 2));
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(data.get(NDArrayIndex.all(), NDArrayIndex.interval(2, 4)), 0.5), Nd4j.getRandom());
                    data.get(NDArrayIndex.all(), NDArrayIndex.interval(4, 6)).assign(Nd4j.rand(minibatch, 2));
                    break;
                default:
                    throw new RuntimeException();
            }
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(0.2).l1(0.3).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).list().layer(0, new VariationalAutoencoder.Builder().nIn(inOutSize).nOut(3).encoderLayerSizes(5).decoderLayerSizes(6).pzxActivationFunction(Activation.TANH).reconstructionDistribution(reconstructionDistributions[i]).activation(new ActivationTanH()).updater(Updater.SGD).build()).pretrain(true).backprop(false).build();
            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();
            mln.initGradientsView();
            mln.fit(data);
            org.deeplearning4j.nn.layers.variational.VariationalAutoencoder layer = (org.deeplearning4j.nn.layers.variational.VariationalAutoencoder) mln.getLayer(0);
            assertFalse(layer.hasLossFunction());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionProb = layer.reconstructionProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionProb.shape());
            Nd4j.getRandom().setSeed(12345);
            INDArray reconstructionLogProb = layer.reconstructionLogProbability(data, 50);
            assertArrayEquals(new int[] { minibatch, 1 }, reconstructionLogProb.shape());
            //                System.out.println(reconstructionDistributions[i]);
            for (int j = 0; j < minibatch; j++) {
                double p = reconstructionProb.getDouble(j);
                double logp = reconstructionLogProb.getDouble(j);
                assertTrue(p >= 0.0 && p <= 1.0);
                assertTrue(logp <= 0.0);
                double pFromLogP = Math.exp(logp);
                assertEquals(p, pFromLogP, 1e-6);
            }
        }
    }
}

Example 2 with BernoulliDistribution

use of org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution in project nd4j by deeplearning4j.

the class SameDiffTests method testPairwiseBooleanTransforms.

@Test
public void testPairwiseBooleanTransforms() {
    /*
        eq, neq, gt, lt, gte, lte, or, and, xor
         */
    // Test transforms (pairwise)
    Nd4j.getRandom().setSeed(12345);
    for (int i = 0; i < 11; i++) {
        SameDiff sd = SameDiff.create();
        int nOut = 4;
        int minibatch = 5;
        INDArray ia = Nd4j.randn(minibatch, nOut);
        INDArray ib = Nd4j.randn(minibatch, nOut);
        SDVariable in1 = sd.var("in1", ia);
        SDVariable in2 = sd.var("in2", ib);
        SDVariable t;
        INDArray expOut;
        switch(i) {
            case 0:
                t = sd.eq(in1, in2);
                expOut = ia.eq(ib);
                break;
            case 1:
                t = sd.neq(in1, in2);
                expOut = ia.neq(ib);
                break;
            case 2:
                t = sd.gt(in1, in2);
                expOut = ia.gt(ib);
                break;
            case 3:
                t = sd.lt(in1, in2);
                expOut = ia.lt(ib);
                break;
            case 4:
                t = sd.gte(in1, in2);
                expOut = ia.dup();
                Nd4j.getExecutioner().exec(new GreaterThanOrEqual(new INDArray[] { ia, ib }, new INDArray[] { expOut }));
                break;
            case 5:
                t = sd.lte(in1, in2);
                expOut = ia.dup();
                Nd4j.getExecutioner().exec(new LessThanOrEqual(new INDArray[] { ia, ib }, new INDArray[] { expOut }));
                break;
            case 6:
                ia = Nd4j.getExecutioner().exec(new BernoulliDistribution(ia, 0.5));
                ib = Nd4j.getExecutioner().exec(new BernoulliDistribution(ib, 0.5));
                t = sd.or(in1, in2);
                expOut = Transforms.or(ia, ib);
                break;
            case 7:
                t = sd.max(in1, in2);
                expOut = Nd4j.getExecutioner().execAndReturn(new OldMax(ia, ib, ia.dup(), ia.length()));
                break;
            case 8:
                t = sd.min(in1, in2);
                expOut = Nd4j.getExecutioner().execAndReturn(new OldMin(ia, ib, ia.dup(), ia.length()));
                break;
            case 9:
                ia = Nd4j.getExecutioner().exec(new BernoulliDistribution(ia, 0.5));
                ib = Nd4j.getExecutioner().exec(new BernoulliDistribution(ib, 0.5));
                t = sd.and(in1, in2);
                expOut = Transforms.and(ia, ib);
                break;
            case 10:
                ia = Nd4j.getExecutioner().exec(new BernoulliDistribution(ia, 0.5));
                ib = Nd4j.getExecutioner().exec(new BernoulliDistribution(ib, 0.5));
                t = sd.xor(in1, in2);
                expOut = Transforms.xor(ia, ib);
                break;
            default:
                throw new RuntimeException();
        }
        log.info("Executing: " + i);
        INDArray out = sd.execAndEndResult();
        assertEquals(expOut, out);
    }
}

Example 3 with BernoulliDistribution

use of org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution in project nd4j by deeplearning4j.

the class UnderSamplingPreProcessorTest method makeDataSetSameL.

/*
        Make a random dataset with 0,1 distribution of classes specified
        Will return as a one-hot vector if twoClass = true
     */
public static DataSet makeDataSetSameL(int batchSize, int timesteps, float[] minorityDist, boolean twoClass) {
    INDArray features = Nd4j.rand(1, batchSize * timesteps * 2).reshape(batchSize, 2, timesteps);
    INDArray labels;
    if (twoClass) {
        labels = Nd4j.zeros(new int[] { batchSize, 2, timesteps });
    } else {
        labels = Nd4j.zeros(new int[] { batchSize, 1, timesteps });
    }
    for (int i = 0; i < batchSize; i++) {
        INDArray l;
        if (twoClass) {
            l = labels.get(NDArrayIndex.point(i), NDArrayIndex.point(1), NDArrayIndex.all());
            Nd4j.getExecutioner().exec(new BernoulliDistribution(l, minorityDist[i]));
            INDArray lOther = labels.get(NDArrayIndex.point(i), NDArrayIndex.point(0), NDArrayIndex.all());
            lOther.assign(Transforms.not(l.dup()));
        } else {
            l = labels.get(NDArrayIndex.point(i), NDArrayIndex.point(0), NDArrayIndex.all());
            Nd4j.getExecutioner().exec(new BernoulliDistribution(l, minorityDist[i]));
        }
    }
    return new DataSet(features, labels);
}

Example 4 with BernoulliDistribution

use of org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution in project nd4j by deeplearning4j.

the class DataSetTest method testMergingWithPerOutputMasking.

@Test
public void testMergingWithPerOutputMasking() {
    // Test 2d mask merging, 2d data
    // features
    INDArray f2d1 = Nd4j.create(new double[] { 1, 2, 3 });
    INDArray f2d2 = Nd4j.create(new double[][] { { 4, 5, 6 }, { 7, 8, 9 } });
    // labels
    INDArray l2d1 = Nd4j.create(new double[] { 1.5, 2.5, 3.5 });
    INDArray l2d2 = Nd4j.create(new double[][] { { 4.5, 5.5, 6.5 }, { 7.5, 8.5, 9.5 } });
    // feature masks
    INDArray fm2d1 = Nd4j.create(new double[] { 0, 1, 1 });
    INDArray fm2d2 = Nd4j.create(new double[][] { { 1, 0, 1 }, { 0, 1, 0 } });
    // label masks
    INDArray lm2d1 = Nd4j.create(new double[] { 1, 1, 0 });
    INDArray lm2d2 = Nd4j.create(new double[][] { { 1, 0, 0 }, { 0, 1, 1 } });
    DataSet mds2d1 = new DataSet(f2d1, l2d1, fm2d1, lm2d1);
    DataSet mds2d2 = new DataSet(f2d2, l2d2, fm2d2, lm2d2);
    DataSet merged = DataSet.merge(Arrays.asList(mds2d1, mds2d2));
    INDArray expFeatures2d = Nd4j.create(new double[][] { { 1, 2, 3 }, { 4, 5, 6 }, { 7, 8, 9 } });
    INDArray expLabels2d = Nd4j.create(new double[][] { { 1.5, 2.5, 3.5 }, { 4.5, 5.5, 6.5 }, { 7.5, 8.5, 9.5 } });
    INDArray expFM2d = Nd4j.create(new double[][] { { 0, 1, 1 }, { 1, 0, 1 }, { 0, 1, 0 } });
    INDArray expLM2d = Nd4j.create(new double[][] { { 1, 1, 0 }, { 1, 0, 0 }, { 0, 1, 1 } });
    DataSet dsExp2d = new DataSet(expFeatures2d, expLabels2d, expFM2d, expLM2d);
    assertEquals(dsExp2d, merged);
    // Test 4d features, 2d labels, 2d masks
    INDArray f4d1 = Nd4j.create(1, 3, 5, 5);
    INDArray f4d2 = Nd4j.create(2, 3, 5, 5);
    DataSet ds4d1 = new DataSet(f4d1, l2d1, null, lm2d1);
    DataSet ds4d2 = new DataSet(f4d2, l2d2, null, lm2d2);
    DataSet merged4d = DataSet.merge(Arrays.asList(ds4d1, ds4d2));
    assertEquals(expLabels2d, merged4d.getLabels());
    assertEquals(expLM2d, merged4d.getLabelsMaskArray());
    // Test 3d mask merging, 3d data
    INDArray f3d1 = Nd4j.create(1, 3, 4);
    INDArray f3d2 = Nd4j.create(1, 3, 3);
    INDArray l3d1 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.create(1, 3, 4), 0.5));
    INDArray l3d2 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.create(2, 3, 3), 0.5));
    INDArray lm3d1 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.create(1, 3, 4), 0.5));
    INDArray lm3d2 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.create(2, 3, 3), 0.5));
    DataSet ds3d1 = new DataSet(f3d1, l3d1, null, lm3d1);
    DataSet ds3d2 = new DataSet(f3d2, l3d2, null, lm3d2);
    INDArray expLabels3d = Nd4j.create(3, 3, 4);
    expLabels3d.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, l3d1);
    expLabels3d.put(new INDArrayIndex[] { NDArrayIndex.interval(1, 2, true), NDArrayIndex.all(), NDArrayIndex.interval(0, 3) }, l3d2);
    INDArray expLM3d = Nd4j.create(3, 3, 4);
    expLM3d.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, lm3d1);
    expLM3d.put(new INDArrayIndex[] { NDArrayIndex.interval(1, 2, true), NDArrayIndex.all(), NDArrayIndex.interval(0, 3) }, lm3d2);
    DataSet merged3d = DataSet.merge(Arrays.asList(ds3d1, ds3d2));
    assertEquals(expLabels3d, merged3d.getLabels());
    assertEquals(expLM3d, merged3d.getLabelsMaskArray());
    // Test 3d features, 2d masks, 2d output (for example: RNN -> global pooling w/ per-output masking)
    DataSet ds3d2d1 = new DataSet(f3d1, l2d1, null, lm2d1);
    DataSet ds3d2d2 = new DataSet(f3d2, l2d2, null, lm2d2);
    DataSet merged3d2d = DataSet.merge(Arrays.asList(ds3d2d1, ds3d2d2));
    assertEquals(expLabels2d, merged3d2d.getLabels());
    assertEquals(expLM2d, merged3d2d.getLabelsMaskArray());
}

Example 5 with BernoulliDistribution

use of org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution in project nd4j by deeplearning4j.

the class GradCheckLoss method testLossWeights2d.

@Test
public void testLossWeights2d() {
    String[] weightTypes = new String[] { "none", "per-example", "per-output", "per-example-output" };
    Nd4j.getRandom().setSeed(12345);
    int nOut = 4;
    int minibatch = 10;
    for (String weightType : weightTypes) {
        for (boolean binary : new boolean[] { true, false }) {
            // Binary mask (like DL4J) or arbitrary weights?
            int[] weightShape;
            switch(weightType) {
                case "none":
                    weightShape = null;
                    break;
                case "per-example":
                    weightShape = new int[] { minibatch, 1 };
                    break;
                case "per-output":
                    weightShape = new int[] { 1, nOut };
                    break;
                case "per-example-output":
                    weightShape = new int[] { minibatch, nOut };
                    break;
                default:
                    throw new RuntimeException("Unknown type: " + weightType);
            }
            INDArray weightArr = null;
            if (!"none".equals(weightType)) {
                if (binary) {
                    weightArr = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(weightShape), 0.5));
                } else {
                    weightArr = Nd4j.rand(weightShape).muli(2.0);
                }
            }
            for (LossFunctions.Reduction reduction : new LossFunctions.Reduction[] { LossFunctions.Reduction.MEAN_BY_COUNT, LossFunctions.Reduction.MEAN_BY_WEIGHT, LossFunctions.Reduction.SUM }) {
                for (String fn : new String[] { "mse", "l1", "l2", "mcxent" }) {
                    SameDiff sd = SameDiff.create();
                    SDVariable input = sd.var("in", new int[] { -1, nOut });
                    SDVariable labels = sd.var("labels", new int[] { -1, nOut });
                    SDVariable weight = null;
                    if (!"none".equals(weightType)) {
                        weight = sd.var("weights", weightArr);
                    }
                    INDArray inputArr = Nd4j.randn(minibatch, nOut).muli(100);
                    INDArray labelsArr = Nd4j.randn(minibatch, nOut).muli(100);
                    LossInfo lossInfo;
                    switch(fn) {
                        case "mse":
                            lossInfo = LossFunctions.mse("out", input, labels, weight, reduction, 1);
                            break;
                        case "l1":
                            lossInfo = LossFunctions.l1("out", input, labels, weight, reduction, 1);
                            // L1 = sum abs error
                            break;
                        case "l2":
                            lossInfo = LossFunctions.l2("out", input, labels, weight, reduction, 1);
                            // L2 = sum squared error
                            break;
                        case "mcxent":
                            lossInfo = LossFunctions.mcxent("out", input, labels, weight, reduction, 1);
                            // mcxent = sum label * log(prob)
                            break;
                        default:
                            throw new RuntimeException();
                    }
                    String msg = "lossFn=" + fn + ", reduction=" + reduction + ", weightType=" + weightType + ", binaryWeight=" + binary;
                    log.info("*** Starting test: " + msg);
                    sd.associateArrayWithVariable(inputArr, input);
                    sd.associateArrayWithVariable(labelsArr, labels);
                    if (weight != null) {
                        sd.associateArrayWithVariable(weightArr, weight);
                    }
                    INDArray out = sd.execAndEndResult();
                    assertEquals(1, out.length());
                    boolean ok = GradCheckUtil.checkGradients(sd);
                    assertTrue(msg, ok);
                }
            }
        }
    }
}