Examples with ClassPathResource org.nd4j.linalg.io.ClassPathResource used on opensource projects

Example 66 with ClassPathResource

use of org.nd4j.linalg.io.ClassPathResource in project deeplearning4j by deeplearning4j.

the class DeepWalkGradientCheck method checkGradients2.

@Test
public void checkGradients2() throws IOException {
    ClassPathResource cpr = new ClassPathResource("graph13.txt");
    int nVertices = 13;
    Graph<String, String> graph = GraphLoader.loadUndirectedGraphEdgeListFile(cpr.getTempFileFromArchive().getAbsolutePath(), 13, ",");
    int vectorSize = 10;
    int windowSize = 3;
    Nd4j.getRandom().setSeed(12345);
    DeepWalk<String, String> deepWalk = new DeepWalk.Builder<String, String>().learningRate(0.01).vectorSize(vectorSize).windowSize(windowSize).learningRate(0.01).build();
    deepWalk.initialize(graph);
    for (int i = 0; i < nVertices; i++) {
        INDArray vector = deepWalk.getVertexVector(i);
        assertArrayEquals(new int[] { 1, vectorSize }, vector.shape());
        System.out.println(Arrays.toString(vector.dup().data().asFloat()));
    }
    GraphWalkIterator<String> iter = new RandomWalkIterator<>(graph, 10);
    deepWalk.fit(iter);
    //Now, to check gradients:
    InMemoryGraphLookupTable table = (InMemoryGraphLookupTable) deepWalk.lookupTable();
    GraphHuffman tree = (GraphHuffman) table.getTree();
    //For each pair of input/output vertices: check gradients
    for (int i = 0; i < nVertices; i++) {
        //in
        //First: check probabilities p(out|in)
        double[] probs = new double[nVertices];
        double sumProb = 0.0;
        for (int j = 0; j < nVertices; j++) {
            probs[j] = table.calculateProb(i, j);
            assertTrue(probs[j] >= 0.0 && probs[j] <= 1.0);
            sumProb += probs[j];
        }
        assertTrue("Output probabilities do not sum to 1.0 (i=" + i + "), sum=" + sumProb, Math.abs(sumProb - 1.0) < 1e-5);
        for (int j = 0; j < nVertices; j++) {
            //out
            //p(j|i)
            int[] pathInnerNodes = tree.getPathInnerNodes(j);
            //Calculate gradients:
            INDArray[][] vecAndGrads = table.vectorsAndGradients(i, j);
            assertEquals(2, vecAndGrads.length);
            assertEquals(pathInnerNodes.length + 1, vecAndGrads[0].length);
            assertEquals(pathInnerNodes.length + 1, vecAndGrads[1].length);
            //Calculate gradients:
            //Two types of gradients to test:
            //(a) gradient of loss fn. wrt inner node vector representation
            //(b) gradient of loss fn. wrt vector for input word
            INDArray vertexVector = table.getVector(i);
            //Check gradients for inner nodes:
            for (int p = 0; p < pathInnerNodes.length; p++) {
                int innerNodeIdx = pathInnerNodes[p];
                INDArray innerNodeVector = table.getInnerNodeVector(innerNodeIdx);
                INDArray innerNodeGrad = vecAndGrads[1][p + 1];
                for (int v = 0; v < innerNodeVector.length(); v++) {
                    double backpropGradient = innerNodeGrad.getDouble(v);
                    double origParamValue = innerNodeVector.getDouble(v);
                    innerNodeVector.putScalar(v, origParamValue + epsilon);
                    double scorePlus = table.calculateScore(i, j);
                    innerNodeVector.putScalar(v, origParamValue - epsilon);
                    double scoreMinus = table.calculateScore(i, j);
                    //reset param so it doesn't affect later calcs
                    innerNodeVector.putScalar(v, origParamValue);
                    double numericalGradient = (scorePlus - scoreMinus) / (2 * epsilon);
                    double relError;
                    if (backpropGradient == 0.0 && numericalGradient == 0.0)
                        relError = 0.0;
                    else {
                        relError = Math.abs(backpropGradient - numericalGradient) / (Math.abs(backpropGradient) + Math.abs(numericalGradient));
                    }
                    String msg = "innerNode grad: i=" + i + ", j=" + j + ", p=" + p + ", v=" + v + " - relError: " + relError + ", scorePlus=" + scorePlus + ", scoreMinus=" + scoreMinus + ", numGrad=" + numericalGradient + ", backpropGrad = " + backpropGradient;
                    if (relError > MAX_REL_ERROR)
                        fail(msg);
                    else
                        System.out.println(msg);
                }
            }
            //Check gradients for input word vector:
            INDArray vectorGrad = vecAndGrads[1][0];
            assertArrayEquals(vectorGrad.shape(), vertexVector.shape());
            for (int v = 0; v < vectorGrad.length(); v++) {
                double backpropGradient = vectorGrad.getDouble(v);
                double origParamValue = vertexVector.getDouble(v);
                vertexVector.putScalar(v, origParamValue + epsilon);
                double scorePlus = table.calculateScore(i, j);
                vertexVector.putScalar(v, origParamValue - epsilon);
                double scoreMinus = table.calculateScore(i, j);
                vertexVector.putScalar(v, origParamValue);
                double numericalGradient = (scorePlus - scoreMinus) / (2 * epsilon);
                double relError;
                if (backpropGradient == 0.0 && numericalGradient == 0.0)
                    relError = 0.0;
                else {
                    relError = Math.abs(backpropGradient - numericalGradient) / (Math.abs(backpropGradient) + Math.abs(numericalGradient));
                }
                String msg = "vector grad: i=" + i + ", j=" + j + ", v=" + v + " - relError: " + relError + ", scorePlus=" + scorePlus + ", scoreMinus=" + scoreMinus + ", numGrad=" + numericalGradient + ", backpropGrad = " + backpropGradient;
                if (relError > MAX_REL_ERROR)
                    fail(msg);
                else
                    System.out.println(msg);
            }
            System.out.println();
        }
    }
}

Example 67 with ClassPathResource

use of org.nd4j.linalg.io.ClassPathResource in project deeplearning4j by deeplearning4j.

the class DeepWalkGradientCheck method checkGradients.

@Test
public void checkGradients() throws IOException {
    ClassPathResource cpr = new ClassPathResource("testgraph_7vertices.txt");
    Graph<String, String> graph = GraphLoader.loadUndirectedGraphEdgeListFile(cpr.getTempFileFromArchive().getAbsolutePath(), 7, ",");
    int vectorSize = 5;
    int windowSize = 2;
    Nd4j.getRandom().setSeed(12345);
    DeepWalk<String, String> deepWalk = new DeepWalk.Builder<String, String>().learningRate(0.01).vectorSize(vectorSize).windowSize(windowSize).learningRate(0.01).build();
    deepWalk.initialize(graph);
    for (int i = 0; i < 7; i++) {
        INDArray vector = deepWalk.getVertexVector(i);
        assertArrayEquals(new int[] { 1, vectorSize }, vector.shape());
        System.out.println(Arrays.toString(vector.dup().data().asFloat()));
    }
    GraphWalkIterator<String> iter = new RandomWalkIterator<>(graph, 8);
    deepWalk.fit(iter);
    //Now, to check gradients:
    InMemoryGraphLookupTable table = (InMemoryGraphLookupTable) deepWalk.lookupTable();
    GraphHuffman tree = (GraphHuffman) table.getTree();
    //For each pair of input/output vertices: check gradients
    for (int i = 0; i < 7; i++) {
        //in
        //First: check probabilities p(out|in)
        double[] probs = new double[7];
        double sumProb = 0.0;
        for (int j = 0; j < 7; j++) {
            probs[j] = table.calculateProb(i, j);
            assertTrue(probs[j] >= 0.0 && probs[j] <= 1.0);
            sumProb += probs[j];
        }
        assertTrue("Output probabilities do not sum to 1.0", Math.abs(sumProb - 1.0) < 1e-5);
        for (int j = 0; j < 7; j++) {
            //out
            //p(j|i)
            int[] pathInnerNodes = tree.getPathInnerNodes(j);
            //Calculate gradients:
            INDArray[][] vecAndGrads = table.vectorsAndGradients(i, j);
            assertEquals(2, vecAndGrads.length);
            assertEquals(pathInnerNodes.length + 1, vecAndGrads[0].length);
            assertEquals(pathInnerNodes.length + 1, vecAndGrads[1].length);
            //Calculate gradients:
            //Two types of gradients to test:
            //(a) gradient of loss fn. wrt inner node vector representation
            //(b) gradient of loss fn. wrt vector for input word
            INDArray vertexVector = table.getVector(i);
            //Check gradients for inner nodes:
            for (int p = 0; p < pathInnerNodes.length; p++) {
                int innerNodeIdx = pathInnerNodes[p];
                INDArray innerNodeVector = table.getInnerNodeVector(innerNodeIdx);
                INDArray innerNodeGrad = vecAndGrads[1][p + 1];
                for (int v = 0; v < innerNodeVector.length(); v++) {
                    double backpropGradient = innerNodeGrad.getDouble(v);
                    double origParamValue = innerNodeVector.getDouble(v);
                    innerNodeVector.putScalar(v, origParamValue + epsilon);
                    double scorePlus = table.calculateScore(i, j);
                    innerNodeVector.putScalar(v, origParamValue - epsilon);
                    double scoreMinus = table.calculateScore(i, j);
                    //reset param so it doesn't affect later calcs
                    innerNodeVector.putScalar(v, origParamValue);
                    double numericalGradient = (scorePlus - scoreMinus) / (2 * epsilon);
                    double relError;
                    if (backpropGradient == 0.0 && numericalGradient == 0.0)
                        relError = 0.0;
                    else {
                        relError = Math.abs(backpropGradient - numericalGradient) / (Math.abs(backpropGradient) + Math.abs(numericalGradient));
                    }
                    String msg = "innerNode grad: i=" + i + ", j=" + j + ", p=" + p + ", v=" + v + " - relError: " + relError + ", scorePlus=" + scorePlus + ", scoreMinus=" + scoreMinus + ", numGrad=" + numericalGradient + ", backpropGrad = " + backpropGradient;
                    if (relError > MAX_REL_ERROR)
                        fail(msg);
                    else
                        System.out.println(msg);
                }
            }
            //Check gradients for input word vector:
            INDArray vectorGrad = vecAndGrads[1][0];
            assertArrayEquals(vectorGrad.shape(), vertexVector.shape());
            for (int v = 0; v < vectorGrad.length(); v++) {
                double backpropGradient = vectorGrad.getDouble(v);
                double origParamValue = vertexVector.getDouble(v);
                vertexVector.putScalar(v, origParamValue + epsilon);
                double scorePlus = table.calculateScore(i, j);
                vertexVector.putScalar(v, origParamValue - epsilon);
                double scoreMinus = table.calculateScore(i, j);
                vertexVector.putScalar(v, origParamValue);
                double numericalGradient = (scorePlus - scoreMinus) / (2 * epsilon);
                double relError;
                if (backpropGradient == 0.0 && numericalGradient == 0.0)
                    relError = 0.0;
                else {
                    relError = Math.abs(backpropGradient - numericalGradient) / (Math.abs(backpropGradient) + Math.abs(numericalGradient));
                }
                String msg = "vector grad: i=" + i + ", j=" + j + ", v=" + v + " - relError: " + relError + ", scorePlus=" + scorePlus + ", scoreMinus=" + scoreMinus + ", numGrad=" + numericalGradient + ", backpropGrad = " + backpropGradient;
                if (relError > MAX_REL_ERROR)
                    fail(msg);
                else
                    System.out.println(msg);
            }
            System.out.println();
        }
    }
}

Example 68 with ClassPathResource

use of org.nd4j.linalg.io.ClassPathResource in project deeplearning4j by deeplearning4j.

the class TestDeepWalk method testBasic.

@Test
public void testBasic() throws IOException {
    //Very basic test. Load graph, build tree, call fit, make sure it doesn't throw any exceptions
    ClassPathResource cpr = new ClassPathResource("testgraph_7vertices.txt");
    Graph<String, String> graph = GraphLoader.loadUndirectedGraphEdgeListFile(cpr.getTempFileFromArchive().getAbsolutePath(), 7, ",");
    int vectorSize = 5;
    int windowSize = 2;
    DeepWalk<String, String> deepWalk = new DeepWalk.Builder<String, String>().learningRate(0.01).vectorSize(vectorSize).windowSize(windowSize).learningRate(0.01).build();
    deepWalk.initialize(graph);
    for (int i = 0; i < 7; i++) {
        INDArray vector = deepWalk.getVertexVector(i);
        assertArrayEquals(new int[] { 1, vectorSize }, vector.shape());
        System.out.println(Arrays.toString(vector.dup().data().asFloat()));
    }
    GraphWalkIterator<String> iter = new RandomWalkIterator<>(graph, 8);
    deepWalk.fit(iter);
    for (int t = 0; t < 5; t++) {
        iter.reset();
        deepWalk.fit(iter);
        System.out.println("--------------------");
        for (int i = 0; i < 7; i++) {
            INDArray vector = deepWalk.getVertexVector(i);
            assertArrayEquals(new int[] { 1, vectorSize }, vector.shape());
            System.out.println(Arrays.toString(vector.dup().data().asFloat()));
        }
    }
}

Example 69 with ClassPathResource

use of org.nd4j.linalg.io.ClassPathResource in project deeplearning4j by deeplearning4j.

the class RegressionTest071 method regressionTestLSTM1.

@Test
public void regressionTestLSTM1() throws Exception {
    File f = new ClassPathResource("regression_testing/071/071_ModelSerializer_Regression_LSTM_1.zip").getTempFileFromArchive();
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(f, true);
    MultiLayerConfiguration conf = net.getLayerWiseConfigurations();
    assertEquals(3, conf.getConfs().size());
    assertTrue(conf.isBackprop());
    assertFalse(conf.isPretrain());
    GravesLSTM l0 = (GravesLSTM) conf.getConf(0).getLayer();
    assertEquals("tanh", l0.getActivationFn().toString());
    assertEquals(3, l0.getNIn());
    assertEquals(4, l0.getNOut());
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l0.getGradientNormalization());
    assertEquals(1.5, l0.getGradientNormalizationThreshold(), 1e-5);
    GravesBidirectionalLSTM l1 = (GravesBidirectionalLSTM) conf.getConf(1).getLayer();
    assertEquals("softsign", l1.getActivationFn().toString());
    assertEquals(4, l1.getNIn());
    assertEquals(4, l1.getNOut());
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l1.getGradientNormalization());
    assertEquals(1.5, l1.getGradientNormalizationThreshold(), 1e-5);
    RnnOutputLayer l2 = (RnnOutputLayer) conf.getConf(2).getLayer();
    assertEquals(4, l2.getNIn());
    assertEquals(5, l2.getNOut());
    assertEquals("softmax", l2.getActivationFn().toString());
    assertTrue(l2.getLossFn() instanceof LossMCXENT);
}

Example 70 with ClassPathResource

use of org.nd4j.linalg.io.ClassPathResource in project deeplearning4j by deeplearning4j.

the class RegressionTest071 method regressionTestMLP2.

@Test
public void regressionTestMLP2() throws Exception {
    File f = new ClassPathResource("regression_testing/071/071_ModelSerializer_Regression_MLP_2.zip").getTempFileFromArchive();
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(f, true);
    MultiLayerConfiguration conf = net.getLayerWiseConfigurations();
    assertEquals(2, conf.getConfs().size());
    assertTrue(conf.isBackprop());
    assertFalse(conf.isPretrain());
    DenseLayer l0 = (DenseLayer) conf.getConf(0).getLayer();
    assertTrue(l0.getActivationFn() instanceof ActivationLReLU);
    assertEquals(3, l0.getNIn());
    assertEquals(4, l0.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l0.getRmsDecay(), 1e-6);
    assertEquals(0.15, l0.getLearningRate(), 1e-6);
    assertEquals(0.6, l0.getDropOut(), 1e-6);
    assertEquals(0.1, l0.getL1(), 1e-6);
    assertEquals(0.2, l0.getL2(), 1e-6);
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l0.getGradientNormalization());
    assertEquals(1.5, l0.getGradientNormalizationThreshold(), 1e-5);
    OutputLayer l1 = (OutputLayer) conf.getConf(1).getLayer();
    assertTrue(l1.getActivationFn() instanceof ActivationIdentity);
    assertEquals(LossFunctions.LossFunction.MSE, l1.getLossFunction());
    assertTrue(l1.getLossFn() instanceof LossMSE);
    assertEquals(4, l1.getNIn());
    assertEquals(5, l1.getNOut());
    assertEquals(WeightInit.DISTRIBUTION, l0.getWeightInit());
    assertEquals(new NormalDistribution(0.1, 1.2), l0.getDist());
    assertEquals(Updater.RMSPROP, l0.getUpdater());
    assertEquals(0.96, l1.getRmsDecay(), 1e-6);
    assertEquals(0.15, l1.getLearningRate(), 1e-6);
    assertEquals(0.6, l1.getDropOut(), 1e-6);
    assertEquals(0.1, l1.getL1(), 1e-6);
    assertEquals(0.2, l1.getL2(), 1e-6);
    assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, l1.getGradientNormalization());
    assertEquals(1.5, l1.getGradientNormalizationThreshold(), 1e-5);
    int numParams = net.numParams();
    assertEquals(Nd4j.linspace(1, numParams, numParams), net.params());
    int updaterSize = net.getUpdater().stateSizeForLayer(net);
    assertEquals(Nd4j.linspace(1, updaterSize, updaterSize), net.getUpdater().getStateViewArray());
}