Examples with SoftMaxDerivative org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative used on opensource projects

Example 1 with SoftMaxDerivative

use of org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative in project nd4j by deeplearning4j.

the class SameDiffTests method testSoftmax.

@Test
public void testSoftmax() {
    SameDiff sameDiff = SameDiff.create();
    INDArray sumInput = Nd4j.linspace(1, 4, 4).reshape(2, 2);
    Map<String, INDArray> inputs = new HashMap<>();
    inputs.put("x", sumInput);
    sameDiff.defineFunction("softmax", new SameDiff.SameDiffFunctionDefinition() {

        @Override
        public SDVariable[] define(SameDiff sameDiff, Map<String, INDArray> inputs, SDVariable[] variableInputs) {
            SDVariable input = sameDiff.var("x", inputs.get("x").dup());
            SDVariable softmax = sameDiff.softmax(input);
            // original shape ends up being 2,2
            return new SDVariable[] { softmax };
        }
    }, inputs);
    INDArray executions = sameDiff.execAndEndResult("softmax");
    INDArray assertions = Transforms.softmax(sumInput.dup());
    assertArrayEquals(sumInput.shape(), executions.shape());
    System.out.println(executions);
    assertEquals(assertions, executions);
    SoftMaxDerivative softMaxDerivative = new SoftMaxDerivative(sumInput);
    Nd4j.getExecutioner().exec(softMaxDerivative);
    System.out.println(softMaxDerivative.z());
}

Example 2 with SoftMaxDerivative

use of org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative in project nd4j by deeplearning4j.

the class CrashTest method op.

protected void op(INDArray x, INDArray y, int i) {
    // broadcast along row & column
    INDArray row = Nd4j.ones(64);
    INDArray column = Nd4j.ones(1024, 1);
    x.addiRowVector(row);
    x.addiColumnVector(column);
    // casual scalar
    x.addi(i * 2);
    // reduction along all dimensions
    float sum = x.sumNumber().floatValue();
    // index reduction
    Nd4j.getExecutioner().exec(new IMax(x), Integer.MAX_VALUE);
    // casual transform
    Nd4j.getExecutioner().exec(new Sqrt(x, x));
    // dup
    INDArray x1 = x.dup(x.ordering());
    INDArray x2 = x.dup(x.ordering());
    INDArray x3 = x.dup('c');
    INDArray x4 = x.dup('f');
    // vstack && hstack
    INDArray vstack = Nd4j.vstack(x, x1, x2, x3, x4);
    INDArray hstack = Nd4j.hstack(x, x1, x2, x3, x4);
    // reduce3 call
    Nd4j.getExecutioner().exec(new ManhattanDistance(x, x2));
    // flatten call
    INDArray flat = Nd4j.toFlattened(x, x1, x2, x3, x4);
    // reduction along dimension: row & column
    INDArray max_0 = x.max(0);
    INDArray max_1 = x.max(1);
    // index reduction along dimension: row & column
    INDArray imax_0 = Nd4j.argMax(x, 0);
    INDArray imax_1 = Nd4j.argMax(x, 1);
    // logisoftmax, softmax & softmax derivative
    Nd4j.getExecutioner().exec(new OldSoftMax(x));
    Nd4j.getExecutioner().exec(new SoftMaxDerivative(x));
    Nd4j.getExecutioner().exec(new LogSoftMax(x));
    // BooleanIndexing
    BooleanIndexing.replaceWhere(x, 5f, Conditions.lessThan(8f));
    // assing on view
    BooleanIndexing.assignIf(x, x1, Conditions.greaterThan(-1000000000f));
    // std var along all dimensions
    float std = x.stdNumber().floatValue();
    // std var along row & col
    INDArray xStd_0 = x.std(0);
    INDArray xStd_1 = x.std(1);
    // blas call
    float dot = (float) Nd4j.getBlasWrapper().dot(x, x1);
    // mmul
    for (boolean tA : paramsA) {
        for (boolean tB : paramsB) {
            INDArray xT = tA ? x.dup() : x.dup().transpose();
            INDArray yT = tB ? y.dup() : y.dup().transpose();
            Nd4j.gemm(xT, yT, tA, tB);
        }
    }
    // specially for views, checking here without dup and rollover
    Nd4j.gemm(x, y, false, false);
    System.out.println("Iteration passed: " + i);
}

Example 3 with SoftMaxDerivative

use of org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative in project nd4j by deeplearning4j.

the class DerivativeTests method correctSoftmax.

public static INDArray correctSoftmax(INDArray X) {
    // this is only for X a row vector
    // should return rank 2 matrix diagonal elements are pi*(1-pi)
    // rest are -pi*pj
    INDArray p = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform("softmax", X.dup()));
    INDArray pCol = p.dup().transpose();
    INDArray pipj = pCol.mmul(p);
    pipj.muli(-1);
    // so now pipj is correct except for the diagonal elements
    // which by the way is what our current softmax der gives us
    INDArray diagp = Nd4j.getExecutioner().execAndReturn(new SoftMaxDerivative(X.dup()));
    // ugly for loop to correct diag elements
    for (int i = 0; i < X.length(); i++) {
        pipj.put(i, i, diagp.getDouble(0, i));
    }
    return pipj;
}

Example 4 with SoftMaxDerivative

use of org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative in project nd4j by deeplearning4j.

the class CudaExecutionerTest method testSoftmax1D_1.

@Test
public void testSoftmax1D_1() throws Exception {
    INDArray input1T = Nd4j.create(new double[] { -0.75, 0.58, 0.42, 1.03, -0.61, 0.19, -0.37, -0.40, -1.42, -0.04 });
    INDArray input1 = Nd4j.create(new double[] { -0.75, 0.58, 0.42, 1.03, -0.61, 0.19, -0.37, -0.40, -1.42, -0.04 });
    INDArray input2 = Nd4j.zerosLike(input1);
    Nd4j.copy(input1, input2);
    INDArray output1 = Nd4j.create(1, 10);
    INDArray output1T = Nd4j.create(1, 10);
    System.out.println("FA --------------------");
    Nd4j.getExecutioner().exec(new OldSoftMax(input1, output1));
    Nd4j.getExecutioner().exec(new OldSoftMax(input1T, output1T));
    System.out.println("FB --------------------");
    System.out.println("Softmax = " + output1);
    INDArray output2 = Nd4j.create(1, 10);
    Nd4j.getExecutioner().exec(new SoftMaxDerivative(input2, output2));
    System.out.println("Softmax Derivative = " + output2);
    INDArray assertion1 = Nd4j.create(new double[] { 0.04, 0.16, 0.14, 0.26, 0.05, 0.11, 0.06, 0.06, 0.02, 0.09 });
    assertArrayEquals(assertion1.data().asFloat(), output1.data().asFloat(), 0.01f);
    assertArrayEquals(assertion1.data().asFloat(), output1T.data().asFloat(), 0.01f);
}

Example 5 with SoftMaxDerivative

use of org.nd4j.linalg.api.ops.impl.transforms.SoftMaxDerivative in project nd4j by deeplearning4j.

the class DerivativeTests method testSoftMaxDerivative.

@Test
public void testSoftMaxDerivative() {
    Random r = new Random(12345L);
    int[] mb = new int[] { 10, 2, 1 };
    for (int minibatch : mb) {
        System.out.println("Minibatch size: " + minibatch);
        INDArray z = Nd4j.zeros(minibatch, 5);
        double[][] in = new double[minibatch][5];
        double[][] softmax = new double[minibatch][5];
        double[][] expOut = new double[minibatch][5];
        for (int i = 0; i < minibatch; i++) {
            double rowSumExp = 0.0;
            for (int j = 0; j < 5; j++) {
                in[i][j] = 10 * r.nextDouble();
                z.putScalar(new int[] { i, j }, in[i][j]);
                rowSumExp += FastMath.exp(in[i][j]);
            }
            for (int j = 0; j < 5; j++) {
                softmax[i][j] = FastMath.exp(in[i][j]) / rowSumExp;
                expOut[i][j] = softmax[i][j] * (1.0 - softmax[i][j]);
            }
        }
        INDArray sm = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform("softmax", z.dup()));
        INDArray zPrime = Nd4j.getExecutioner().execAndReturn(new SoftMaxDerivative(z));
        System.out.println(Arrays.toString(sm.data().asDouble()));
        System.out.println(Arrays.toString(zPrime.data().asDouble()));
        assertNotEquals(sm, zPrime);
        for (int i = 0; i < minibatch; i++) {
            for (int j = 0; j < 5; j++) {
                double relError = Math.abs(expOut[i][j] - zPrime.getDouble(i, j)) / (Math.abs(expOut[i][j]) + Math.abs(zPrime.getDouble(i, j)));
                // System.out.println("Error: " + relError);
                assertTrue(relError < REL_ERROR_TOLERANCE);
            }
        }
    }
}