Examples with ComputationGraph org.deeplearning4j.nn.graph.ComputationGraph used on opensource projects

Example 6 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class FlowIterationListener method buildModelInfo.

protected ModelInfo buildModelInfo(Model model) {
    ModelInfo modelInfo = new ModelInfo();
    if (model instanceof ComputationGraph) {
        ComputationGraph graph = (ComputationGraph) model;
        /*
                we assume that graph starts on input. every layer connected to input - is on y1
                every layer connected to y1, is on y2 etc.
              */
        List<String> inputs = graph.getConfiguration().getNetworkInputs();
        // now we need to add inputs as y0 nodes
        int x = 0;
        for (String input : inputs) {
            GraphVertex vertex = graph.getVertex(input);
            long numSamples;
            long tadLength;
            if (vertex.getInputs() == null || vertex.getInputs().length == 0) {
                numSamples = 0;
                tadLength = 0;
            } else {
                INDArray gInput = vertex.getInputs()[0];
                tadLength = Shape.getTADLength(gInput.shape(), ArrayUtil.range(1, gInput.rank()));
                numSamples = gInput.lengthLong() / tadLength;
            }
            StringBuilder builder = new StringBuilder();
            builder.append("Vertex name: ").append(input).append("<br/>");
            builder.append("Model input").append("<br/>");
            builder.append("Input size: ").append(tadLength).append("<br/>");
            builder.append("Batch size: ").append(numSamples).append("<br/>");
            LayerInfo info = new LayerInfo();
            info.setId(0);
            info.setName(input);
            info.setY(0);
            info.setX(x);
            info.setLayerType(INPUT);
            info.setDescription(new Description());
            info.getDescription().setMainLine("Model input");
            info.getDescription().setText(builder.toString());
            modelInfo.addLayer(info);
            x++;
        }
        GraphVertex[] vertices = graph.getVertices();
        // filling grid in LTR/TTB direction
        List<String> needle = new ArrayList<>();
        // we assume that max row can't be higher then total number of vertices
        for (int y = 1; y < vertices.length; y++) {
            if (needle.isEmpty())
                needle.addAll(inputs);
            /*
                    for each grid row we look for nodes, that are connected to previous layer
                */
            List<LayerInfo> layersForGridY = flattenToY(modelInfo, vertices, needle, y);
            needle.clear();
            for (LayerInfo layerInfo : layersForGridY) {
                needle.add(layerInfo.getName());
            }
            if (needle.isEmpty())
                break;
        }
    } else if (model instanceof MultiLayerNetwork) {
        MultiLayerNetwork network = (MultiLayerNetwork) model;
        // manually adding input layer
        INDArray input = model.input();
        long tadLength = Shape.getTADLength(input.shape(), ArrayUtil.range(1, input.rank()));
        long numSamples = input.lengthLong() / tadLength;
        StringBuilder builder = new StringBuilder();
        builder.append("Model input").append("<br/>");
        builder.append("Input size: ").append(tadLength).append("<br/>");
        builder.append("Batch size: ").append(numSamples).append("<br/>");
        LayerInfo info = new LayerInfo();
        info.setId(0);
        info.setName("Input");
        info.setY(0);
        info.setX(0);
        info.setLayerType(INPUT);
        info.setDescription(new Description());
        info.getDescription().setMainLine("Model input");
        info.getDescription().setText(builder.toString());
        info.addConnection(0, 1);
        modelInfo.addLayer(info);
        // entry 0 is reserved for inputs
        int y = 1;
        // for MLN x value is always 0
        final int x = 0;
        for (Layer layer : network.getLayers()) {
            LayerInfo layerInfo = getLayerInfo(layer, x, y, y);
            // since it's MLN, we know connections in advance as curLayer + 1
            layerInfo.addConnection(x, y + 1);
            modelInfo.addLayer(layerInfo);
            y++;
        }
        LayerInfo layerInfo = modelInfo.getLayerInfoByCoords(x, y - 1);
        layerInfo.dropConnections();
    }
    // find layers without connections, and mark them as output layers
    for (LayerInfo layerInfo : modelInfo.getLayers()) {
        if (layerInfo.getConnections().size() == 0)
            layerInfo.setLayerType("OUTPUT");
    }
    // now we apply colors to distinct layer types
    AtomicInteger cnt = new AtomicInteger(0);
    for (String layerType : modelInfo.getLayerTypes()) {
        String curColor = colors.get(cnt.getAndIncrement());
        if (cnt.get() >= colors.size())
            cnt.set(0);
        for (LayerInfo layerInfo : modelInfo.getLayersByType(layerType)) {
            if (layerType.equals(INPUT)) {
                layerInfo.setColor("#99ff66");
            } else if (layerType.equals("OUTPUT")) {
                layerInfo.setColor("#e6e6e6");
            } else {
                layerInfo.setColor(curColor);
            }
        }
    }
    return modelInfo;
}

Example 7 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class FlowIterationListener method buildModelState.

protected void buildModelState(Model model) {
    // first we update performance state
    long timeSpent = currTime - lastTime;
    float timeSec = timeSpent / 1000f;
    INDArray input = model.input();
    long tadLength = Shape.getTADLength(input.shape(), ArrayUtil.range(1, input.rank()));
    long numSamples = input.lengthLong() / tadLength;
    modelState.addPerformanceSamples(numSamples / timeSec);
    modelState.addPerformanceBatches(1 / timeSec);
    modelState.setIterationTime(timeSpent);
    // now model score
    modelState.addScore((float) model.score());
    modelState.setScore((float) model.score());
    modelState.setTrainingTime(parseTime(System.currentTimeMillis() - initTime));
    // and now update model params/gradients
    Map<String, Map> newGrad = new LinkedHashMap<>();
    Map<String, Map> newParams = new LinkedHashMap<>();
    Map<String, INDArray> params = model.paramTable();
    Layer[] layers = null;
    if (model instanceof MultiLayerNetwork) {
        layers = ((MultiLayerNetwork) model).getLayers();
    } else if (model instanceof ComputationGraph) {
        layers = ((ComputationGraph) model).getLayers();
    }
    List<Double> lrs = new ArrayList<>();
    if (layers != null) {
        for (Layer layer : layers) {
            lrs.add(layer.conf().getLayer().getLearningRate());
        }
        modelState.setLearningRates(lrs);
    }
    Map<Integer, LayerParams> layerParamsMap = new LinkedHashMap<>();
    for (Map.Entry<String, INDArray> entry : params.entrySet()) {
        String param = entry.getKey();
        if (!Character.isDigit(param.charAt(0)))
            continue;
        int layer = Integer.parseInt(param.replaceAll("\\_.*$", ""));
        String key = param.replaceAll("^.*?_", "").toLowerCase();
        if (!layerParamsMap.containsKey(layer))
            layerParamsMap.put(layer, new LayerParams());
        HistogramBin histogram = new HistogramBin.Builder(entry.getValue().dup()).setBinCount(14).setRounding(6).build();
        // TODO: something better would be nice to have here
        if (key.equalsIgnoreCase("w")) {
            layerParamsMap.get(layer).setW(histogram.getData());
        } else if (key.equalsIgnoreCase("rw")) {
            layerParamsMap.get(layer).setRW(histogram.getData());
        } else if (key.equalsIgnoreCase("rwf")) {
            layerParamsMap.get(layer).setRWF(histogram.getData());
        } else if (key.equalsIgnoreCase("b")) {
            layerParamsMap.get(layer).setB(histogram.getData());
        }
    }
    modelState.setLayerParams(layerParamsMap);
}

Example 8 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestRenders method testHistogramComputationGraphUnderscoresInName.

@Test
public void testHistogramComputationGraphUnderscoresInName() throws Exception {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input").setInputTypes(InputType.convolutional(1, 28, 28)).addLayer("cnn_1", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(3).build(), "input").addLayer("cnn_2", new ConvolutionLayer.Builder(4, 4).stride(2, 2).padding(1, 1).nIn(1).nOut(3).build(), "input").addLayer("max_1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).build(), "cnn_1", "cnn_2").addLayer("output", new OutputLayer.Builder().nIn(7 * 7 * 6).nOut(10).build(), "max_1").setOutputs("output").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    graph.setListeners(new HistogramIterationListener(1), new ScoreIterationListener(1));
    DataSetIterator mnist = new MnistDataSetIterator(32, 640, false, true, false, 12345);
    graph.fit(mnist);
}

Example 9 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class TestFlowListener method testUICG.

@Test
public void testUICG() throws Exception {
    // Number of input channels
    int nChannels = 1;
    // The number of possible outcomes
    int outputNum = 10;
    // Test batch size
    int batchSize = 64;
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);
    ComputationGraphConfiguration conf = // Training iterations as above
    new NeuralNetConfiguration.Builder().seed(12345).iterations(1).regularization(true).l2(0.0005).learningRate(0.01).weightInit(WeightInit.XAVIER).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.NESTEROVS).momentum(0.9).graphBuilder().addInputs("in").addLayer("0", new ConvolutionLayer.Builder(5, 5).nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(), "in").addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "0").addLayer("2", new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1").addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "2").addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3").addLayer("5", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).activation(Activation.SOFTMAX).build(), "4").setOutputs("5").setInputTypes(InputType.convolutionalFlat(28, 28, 1)).backprop(true).pretrain(false).build();
    ComputationGraph net = new ComputationGraph(conf);
    net.init();
    net.setListeners(new FlowIterationListener(1), new ScoreIterationListener(1));
    for (int i = 0; i < 50; i++) {
        net.fit(mnistTrain.next());
        Thread.sleep(1000);
    }
    Thread.sleep(100000);
}

Example 10 with ComputationGraph

use of org.deeplearning4j.nn.graph.ComputationGraph in project deeplearning4j by deeplearning4j.

the class GradientCheckTestsComputationGraph method testBasicTwoOutputs.

@Test
public void testBasicTwoOutputs() {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(Updater.NONE).learningRate(1.0).graphBuilder().addInputs("in1", "in2").addLayer("d0", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in1").addLayer("d1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "in2").addLayer("out1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(2).nOut(2).activation(Activation.IDENTITY).build(), "d0").addLayer("out2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.L2).nIn(2).nOut(2).activation(Activation.IDENTITY).build(), "d1").setOutputs("out1", "out2").pretrain(false).backprop(true).build();
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    System.out.println("Num layers: " + graph.getNumLayers());
    System.out.println("Num params: " + graph.numParams());
    Nd4j.getRandom().setSeed(12345);
    int nParams = graph.numParams();
    INDArray newParams = Nd4j.rand(1, nParams);
    graph.setParams(newParams);
    int[] mbSizes = new int[] { 1, 3, 10 };
    for (int minibatch : mbSizes) {
        INDArray in1 = Nd4j.rand(minibatch, 2);
        INDArray in2 = Nd4j.rand(minibatch, 2);
        INDArray labels1 = Nd4j.rand(minibatch, 2);
        INDArray labels2 = Nd4j.rand(minibatch, 2);
        String testName = "testBasicStackUnstack() - minibatch = " + minibatch;
        if (PRINT_RESULTS) {
            System.out.println(testName);
            for (int j = 0; j < graph.getNumLayers(); j++) System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }
        boolean gradOK = GradientCheckUtil.checkGradients(graph, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, new INDArray[] { in1, in2 }, new INDArray[] { labels1, labels2 });
        assertTrue(testName, gradOK);
    }
}