Examples with BroadcastAddOp org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp used on opensource projects

Example 6 with BroadcastAddOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp in project deeplearning4j by deeplearning4j.

the class BatchNormalizationTest method testCnnForwardBackward.

@Test
public void testCnnForwardBackward() {
    double eps = 1e-5;
    int nIn = 4;
    int hw = 3;
    int minibatch = 2;
    Nd4j.getRandom().setSeed(12345);
    INDArray input = Nd4j.rand('c', new int[] { minibatch, nIn, hw, hw });
    //TODO: other values for gamma/beta
    INDArray gamma = Nd4j.ones(1, nIn);
    INDArray beta = Nd4j.zeros(1, nIn);
    Layer l = getLayer(nIn, eps, false, -1, -1);
    INDArray mean = input.mean(0, 2, 3);
    INDArray var = input.var(false, 0, 2, 3);
    INDArray xHat = Nd4j.getExecutioner().execAndReturn(new BroadcastSubOp(input, mean, input.dup(), 1));
    Nd4j.getExecutioner().execAndReturn(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1));
    INDArray outExpected = Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1));
    Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(outExpected, beta, outExpected, 1));
    INDArray out = l.activate(input, true);
    System.out.println(Arrays.toString(outExpected.data().asDouble()));
    System.out.println(Arrays.toString(out.data().asDouble()));
    assertEquals(outExpected, out);
    //-------------------------------------------------------------
    //Check backprop
    //dL/dy
    INDArray epsilon = Nd4j.rand('c', new int[] { minibatch, nIn, hw, hw });
    int effectiveMinibatch = minibatch * hw * hw;
    INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3);
    INDArray dldbetaExp = epsilon.sum(0, 2, 3);
    //epsilon.mulRowVector(gamma);
    INDArray dldxhat = Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1));
    INDArray inputSubMean = Nd4j.getExecutioner().execAndReturn(new BroadcastSubOp(input, mean, input.dup(), 1));
    INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5);
    dldvar = Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1));
    dldvar = dldvar.sum(0, 2, 3);
    INDArray dldmu = Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1)).neg().sum(0, 2, 3);
    dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch)));
    INDArray dldinExp = Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1));
    dldinExp = dldinExp.add(Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1)));
    dldinExp = Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1));
    Pair<Gradient, INDArray> p = l.backpropGradient(epsilon);
    INDArray dldgamma = p.getFirst().getGradientFor("gamma");
    INDArray dldbeta = p.getFirst().getGradientFor("beta");
    assertEquals(dldgammaExp, dldgamma);
    assertEquals(dldbetaExp, dldbeta);
    //        System.out.println("EPSILONS");
    //        System.out.println(Arrays.toString(dldinExp.data().asDouble()));
    //        System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
    assertEquals(dldinExp, p.getSecond());
}

Example 7 with BroadcastAddOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp in project deeplearning4j by deeplearning4j.

the class MaskedReductionUtil method maskedPoolingEpsilonTimeSeries.

public static INDArray maskedPoolingEpsilonTimeSeries(PoolingType poolingType, INDArray input, INDArray mask, INDArray epsilon2d, int pnorm) {
    if (input.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 input activation array: got " + input.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }
    if (epsilon2d.rank() != 2) {
        throw new IllegalArgumentException("Expected rank 2 array for errors: got " + epsilon2d.rank());
    }
    switch(poolingType) {
        case MAX:
            //TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
            INDArray negInfMask = Transforms.not(mask);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
            INDArray withInf = Nd4j.createUninitialized(input.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(input, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
            INDArray isMax = Nd4j.getExecutioner().execAndReturn(new IsMax(withInf, 2));
            return Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(isMax, epsilon2d, isMax, 0, 1));
        case AVG:
        case SUM:
            //if out = sum(in,dims) then dL/dIn = dL/dOut -> duplicate to each step and mask
            //if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
            //With masking: N differs for different time series
            INDArray out = Nd4j.createUninitialized(input.shape(), 'f');
            //Broadcast copy op, then divide and mask to 0 as appropriate
            Nd4j.getExecutioner().exec(new BroadcastCopyOp(out, epsilon2d, out, 0, 1));
            Nd4j.getExecutioner().exec(new BroadcastMulOp(out, mask, out, 0, 2));
            if (poolingType == PoolingType.SUM) {
                return out;
            }
            //[minibatchSize,tsLength] -> [minibatchSize,1]
            INDArray nEachTimeSeries = mask.sum(1);
            Nd4j.getExecutioner().exec(new BroadcastDivOp(out, nEachTimeSeries, out, 0));
            return out;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(input.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(input, mask, masked2, 0, 2));
            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = Transforms.pow(abs.sum(2), 1.0 / pnorm);
            INDArray numerator;
            if (pnorm == 2) {
                numerator = input.dup();
            } else {
                INDArray absp2 = Transforms.pow(Transforms.abs(input, true), pnorm - 2, false);
                numerator = input.mul(absp2);
            }
            INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
            denom.rdivi(epsilon2d);
            Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, 0, 1));
            //Apply mask
            Nd4j.getExecutioner().exec(new BroadcastMulOp(numerator, mask, numerator, 0, 2));
            return numerator;
        case NONE:
            throw new UnsupportedOperationException("NONE pooling type not supported");
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}

Example 8 with BroadcastAddOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp in project nd4j by deeplearning4j.

the class StandardizeStrategy method revert.

/**
 * Denormalize a data array
 *
 * @param array the data to denormalize
 * @param stats statistics of the data population
 */
@Override
public void revert(INDArray array, INDArray maskArray, DistributionStats stats) {
    if (array.rank() <= 2) {
        array.muliRowVector(filteredStd(stats));
        array.addiRowVector(stats.getMean());
    } else {
        Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, filteredStd(stats), array, 1));
        Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getMean(), array, 1));
    }
    if (maskArray != null) {
        DataSetUtil.setMaskedValuesToZero(array, maskArray);
    }
}

Example 9 with BroadcastAddOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp in project nd4j by deeplearning4j.

the class MinMaxStrategy method revert.

/**
 * Denormalize a data array
 *
 * @param array the data to denormalize
 * @param stats statistics of the data population
 */
@Override
public void revert(INDArray array, INDArray maskArray, MinMaxStats stats) {
    // Subtract target range minimum value
    array.subi(minRange);
    // Scale by target range
    array.divi(maxRange - minRange);
    if (array.rank() <= 2) {
        array.muliRowVector(stats.getRange());
        array.addiRowVector(stats.getLower());
    } else {
        Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(array, stats.getRange(), array, 1));
        Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(array, stats.getLower(), array, 1));
    }
    if (maskArray != null) {
        DataSetUtil.setMaskedValuesToZero(array, maskArray);
    }
}