Search in sources :

Example 6 with BroadcastDivOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp in project deeplearning4j by deeplearning4j.

the class L2NormalizeVertex method doBackward.

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt) {
    if (!canDoBackward())
        throw new IllegalStateException("Cannot do backward pass: errors not set (L2NormalizeVertex " + vertexName + " idx " + vertexIndex + ")");
    INDArray x = inputs[0];
    int[] dimensions = getDimensions(x);
    INDArray norm = x.norm2(dimensions);
    INDArray norm3 = Transforms.pow(norm, 3.0, true);
    // in case of div/0
    Transforms.max(norm, eps, false);
    Transforms.max(norm3, eps, false);
    INDArray dLdx;
    if (x.rank() == 2) {
        // 2D case
        dLdx = epsilon.divColumnVector(norm);
        INDArray xDivNorm3 = x.divColumnVector(norm3);
        dLdx.subi(xDivNorm3.muliColumnVector(epsilon.mul(x).sum(1)));
    } else {
        //RNN and CNN case - Broadcast along dimension 0
        INDArray dx = epsilon.mul(x).sum(dimensions);
        //x / |x|_2^3 * sum_k (dLda*x)
        INDArray xDivNorm3 = Nd4j.createUninitialized(x.shape(), x.ordering());
        Nd4j.getExecutioner().exec(new BroadcastDivOp(x, norm3, xDivNorm3, 0));
        Nd4j.getExecutioner().exec(new BroadcastMulOp(xDivNorm3, dx, xDivNorm3, 0));
        //1/|x|_2 * dLda - above
        dLdx = Nd4j.createUninitialized(epsilon.shape(), epsilon.ordering());
        Nd4j.getExecutioner().exec(new BroadcastDivOp(epsilon, norm, dLdx, 0));
        dLdx.subi(xDivNorm3);
    }
    return new Pair<>(null, new INDArray[] { dLdx });
}
Also used : INDArray(org.nd4j.linalg.api.ndarray.INDArray) BroadcastMulOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp) BroadcastDivOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp) Pair(org.deeplearning4j.berkeley.Pair)

Example 7 with BroadcastDivOp

use of org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp in project deeplearning4j by deeplearning4j.

the class MaskedReductionUtil method maskedPoolingEpsilonTimeSeries.

public static INDArray maskedPoolingEpsilonTimeSeries(PoolingType poolingType, INDArray input, INDArray mask, INDArray epsilon2d, int pnorm) {
    if (input.rank() != 3) {
        throw new IllegalArgumentException("Expect rank 3 input activation array: got " + input.rank());
    }
    if (mask.rank() != 2) {
        throw new IllegalArgumentException("Expect rank 2 array for mask: got " + mask.rank());
    }
    if (epsilon2d.rank() != 2) {
        throw new IllegalArgumentException("Expected rank 2 array for errors: got " + epsilon2d.rank());
    }
    switch(poolingType) {
        case MAX:
            //TODO This is ugly - replace it with something better... Need something like a Broadcast CAS op
            INDArray negInfMask = Transforms.not(mask);
            BooleanIndexing.replaceWhere(negInfMask, Double.NEGATIVE_INFINITY, Conditions.equals(1.0));
            INDArray withInf = Nd4j.createUninitialized(input.shape());
            Nd4j.getExecutioner().exec(new BroadcastAddOp(input, negInfMask, withInf, 0, 2));
            //At this point: all the masked out steps have value -inf, hence can't be the output of the MAX op
            INDArray isMax = Nd4j.getExecutioner().execAndReturn(new IsMax(withInf, 2));
            return Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(isMax, epsilon2d, isMax, 0, 1));
        case AVG:
        case SUM:
            //if out = sum(in,dims) then dL/dIn = dL/dOut -> duplicate to each step and mask
            //if out = avg(in,dims) then dL/dIn = 1/N * dL/dOut
            //With masking: N differs for different time series
            INDArray out = Nd4j.createUninitialized(input.shape(), 'f');
            //Broadcast copy op, then divide and mask to 0 as appropriate
            Nd4j.getExecutioner().exec(new BroadcastCopyOp(out, epsilon2d, out, 0, 1));
            Nd4j.getExecutioner().exec(new BroadcastMulOp(out, mask, out, 0, 2));
            if (poolingType == PoolingType.SUM) {
                return out;
            }
            //[minibatchSize,tsLength] -> [minibatchSize,1]
            INDArray nEachTimeSeries = mask.sum(1);
            Nd4j.getExecutioner().exec(new BroadcastDivOp(out, nEachTimeSeries, out, 0));
            return out;
        case PNORM:
            //Similar to average and sum pooling: there's no N term here, so we can just set the masked values to 0
            INDArray masked2 = Nd4j.createUninitialized(input.shape());
            Nd4j.getExecutioner().exec(new BroadcastMulOp(input, mask, masked2, 0, 2));
            INDArray abs = Transforms.abs(masked2, true);
            Transforms.pow(abs, pnorm, false);
            INDArray pNorm = Transforms.pow(abs.sum(2), 1.0 / pnorm);
            INDArray numerator;
            if (pnorm == 2) {
                numerator = input.dup();
            } else {
                INDArray absp2 = Transforms.pow(Transforms.abs(input, true), pnorm - 2, false);
                numerator = input.mul(absp2);
            }
            INDArray denom = Transforms.pow(pNorm, pnorm - 1, false);
            denom.rdivi(epsilon2d);
            Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(numerator, denom, numerator, 0, 1));
            //Apply mask
            Nd4j.getExecutioner().exec(new BroadcastMulOp(numerator, mask, numerator, 0, 2));
            return numerator;
        case NONE:
            throw new UnsupportedOperationException("NONE pooling type not supported");
        default:
            throw new UnsupportedOperationException("Unknown or not supported pooling type: " + poolingType);
    }
}
Also used : IsMax(org.nd4j.linalg.api.ops.impl.transforms.IsMax) BroadcastAddOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp) INDArray(org.nd4j.linalg.api.ndarray.INDArray) BroadcastMulOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp) BroadcastCopyOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastCopyOp) BroadcastDivOp(org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp)

Aggregations

INDArray (org.nd4j.linalg.api.ndarray.INDArray)7 BroadcastDivOp (org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp)7 BroadcastMulOp (org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp)6 BroadcastAddOp (org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp)5 Pair (org.deeplearning4j.berkeley.Pair)2 Gradient (org.deeplearning4j.nn.gradient.Gradient)2 BroadcastCopyOp (org.nd4j.linalg.api.ops.impl.broadcast.BroadcastCopyOp)2 BroadcastSubOp (org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp)2 IsMax (org.nd4j.linalg.api.ops.impl.transforms.IsMax)2 Layer (org.deeplearning4j.nn.api.Layer)1 DefaultGradient (org.deeplearning4j.nn.gradient.DefaultGradient)1 Test (org.junit.Test)1