Examples with InputPreProcessor org.deeplearning4j.nn.conf.InputPreProcessor used on opensource projects

Example 1 with InputPreProcessor

use of org.deeplearning4j.nn.conf.InputPreProcessor in project deeplearning4j by deeplearning4j.

the class KerasModel method getComputationGraphConfiguration.

/**
     * Configure a ComputationGraph from this Keras Model configuration.
     *
     * @return          ComputationGraph
     */
public ComputationGraphConfiguration getComputationGraphConfiguration() throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    if (!this.className.equals(MODEL_CLASS_NAME_MODEL) && !this.className.equals(MODEL_CLASS_NAME_SEQUENTIAL))
        throw new InvalidKerasConfigurationException("Keras model class name " + this.className + " incompatible with ComputationGraph");
    NeuralNetConfiguration.Builder modelBuilder = new NeuralNetConfiguration.Builder();
    ComputationGraphConfiguration.GraphBuilder graphBuilder = modelBuilder.graphBuilder();
    /* Build String array of input layer names, add to ComputationGraph. */
    String[] inputLayerNameArray = new String[this.inputLayerNames.size()];
    this.inputLayerNames.toArray(inputLayerNameArray);
    graphBuilder.addInputs(inputLayerNameArray);
    /* Build InputType array of input layer types, add to ComputationGraph. */
    List<InputType> inputTypeList = new ArrayList<InputType>();
    for (String inputLayerName : this.inputLayerNames) inputTypeList.add(this.layers.get(inputLayerName).getOutputType());
    InputType[] inputTypes = new InputType[inputTypeList.size()];
    inputTypeList.toArray(inputTypes);
    graphBuilder.setInputTypes(inputTypes);
    /* Build String array of output layer names, add to ComputationGraph. */
    String[] outputLayerNameArray = new String[this.outputLayerNames.size()];
    this.outputLayerNames.toArray(outputLayerNameArray);
    graphBuilder.setOutputs(outputLayerNameArray);
    Map<String, InputPreProcessor> preprocessors = new HashMap<String, InputPreProcessor>();
    /* Add layersOrdered one at a time. */
    for (KerasLayer layer : this.layersOrdered) {
        /* Get inbound layer names. */
        List<String> inboundLayerNames = layer.getInboundLayerNames();
        String[] inboundLayerNamesArray = new String[inboundLayerNames.size()];
        inboundLayerNames.toArray(inboundLayerNamesArray);
        /* Get inbound InputTypes and InputPreProcessor, if necessary. */
        List<InputType> inboundTypeList = new ArrayList<InputType>();
        for (String layerName : inboundLayerNames) inboundTypeList.add(this.outputTypes.get(layerName));
        InputType[] inboundTypeArray = new InputType[inboundTypeList.size()];
        inboundTypeList.toArray(inboundTypeArray);
        InputPreProcessor preprocessor = layer.getInputPreprocessor(inboundTypeArray);
        if (layer.usesRegularization())
            modelBuilder.setUseRegularization(true);
        if (layer.isLayer()) {
            /* Add DL4J layer. */
            if (preprocessor != null)
                preprocessors.put(layer.getLayerName(), preprocessor);
            graphBuilder.addLayer(layer.getLayerName(), layer.getLayer(), inboundLayerNamesArray);
            if (this.outputLayerNames.contains(layer.getLayerName()) && !(layer.getLayer() instanceof IOutputLayer))
                log.warn("Model cannot be trained: output layer " + layer.getLayerName() + " is not an IOutputLayer (no loss function specified)");
        } else if (layer.isVertex()) {
            /* Add DL4J vertex. */
            if (preprocessor != null)
                preprocessors.put(layer.getLayerName(), preprocessor);
            graphBuilder.addVertex(layer.getLayerName(), layer.getVertex(), inboundLayerNamesArray);
            if (this.outputLayerNames.contains(layer.getLayerName()) && !(layer.getVertex() instanceof IOutputLayer))
                log.warn("Model cannot be trained: output vertex " + layer.getLayerName() + " is not an IOutputLayer (no loss function specified)");
        } else if (layer.isInputPreProcessor()) {
            if (preprocessor == null)
                throw new UnsupportedKerasConfigurationException("Layer " + layer.getLayerName() + " could not be mapped to Layer, Vertex, or InputPreProcessor");
            graphBuilder.addVertex(layer.getLayerName(), new PreprocessorVertex(preprocessor), inboundLayerNamesArray);
        }
        if (this.outputLayerNames.contains(layer.getLayerName()))
            log.warn("Model cannot be trained: output " + layer.getLayerName() + " is not an IOutputLayer (no loss function specified)");
    }
    graphBuilder.setInputPreProcessors(preprocessors);
    /* Whether to use standard backprop (or BPTT) or truncated BPTT. */
    if (this.useTruncatedBPTT && this.truncatedBPTT > 0)
        graphBuilder.backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(truncatedBPTT).tBPTTBackwardLength(truncatedBPTT);
    else
        graphBuilder.backpropType(BackpropType.Standard);
    return graphBuilder.build();
}

Example 2 with InputPreProcessor

use of org.deeplearning4j.nn.conf.InputPreProcessor in project deeplearning4j by deeplearning4j.

the class TestPreProcessors method testRnnToCnnPreProcessor.

@Test
public void testRnnToCnnPreProcessor() {
    //Two ways to test this:
    // (a) check that doing preProcess + backprop on a given input gives same result
    // (b) compare to ComposableInputPreProcessor(CNNtoFF, FFtoRNN)
    int[] miniBatchSizes = { 5, 1 };
    int[] timeSeriesLengths = { 9, 1 };
    int[] inputHeights = { 10, 30 };
    int[] inputWidths = { 10, 30 };
    int[] numChannels = { 1, 3, 6 };
    int cnnNChannelsIn = 3;
    Nd4j.getRandom().setSeed(12345);
    System.out.println();
    for (int miniBatchSize : miniBatchSizes) {
        for (int timeSeriesLength : timeSeriesLengths) {
            for (int inputHeight : inputHeights) {
                for (int inputWidth : inputWidths) {
                    for (int nChannels : numChannels) {
                        InputPreProcessor proc = new RnnToCnnPreProcessor(inputHeight, inputWidth, nChannels);
                        NeuralNetConfiguration nnc = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(inputWidth, inputHeight).nIn(cnnNChannelsIn).nOut(nChannels).build()).build();
                        int numParams = nnc.getLayer().initializer().numParams(nnc);
                        INDArray params = Nd4j.create(1, numParams);
                        ConvolutionLayer layer = (ConvolutionLayer) nnc.getLayer().instantiate(nnc, null, 0, params, true);
                        layer.setInputMiniBatchSize(miniBatchSize);
                        int[] shape_rnn = new int[] { miniBatchSize, nChannels * inputHeight * inputWidth, timeSeriesLength };
                        INDArray rand = Nd4j.rand(shape_rnn);
                        INDArray activationsRnn_c = Nd4j.create(shape_rnn, 'c');
                        INDArray activationsRnn_f = Nd4j.create(shape_rnn, 'f');
                        activationsRnn_c.assign(rand);
                        activationsRnn_f.assign(rand);
                        assertEquals(activationsRnn_c, activationsRnn_f);
                        //Check shape of outputs:
                        INDArray activationsCnn_c = proc.preProcess(activationsRnn_c, miniBatchSize);
                        INDArray activationsCnn_f = proc.preProcess(activationsRnn_f, miniBatchSize);
                        int[] shape_cnn = new int[] { miniBatchSize * timeSeriesLength, nChannels, inputHeight, inputWidth };
                        assertArrayEquals(shape_cnn, activationsCnn_c.shape());
                        assertArrayEquals(shape_cnn, activationsCnn_f.shape());
                        assertEquals(activationsCnn_c, activationsCnn_f);
                        //Check backward pass. Given that activations and epsilons have same shape, they should
                        //be opposite operations - i.e., get the same thing back out
                        INDArray twiceProcessed_c = proc.backprop(activationsCnn_c, miniBatchSize);
                        INDArray twiceProcessed_f = proc.backprop(activationsCnn_c, miniBatchSize);
                        assertArrayEquals(shape_rnn, twiceProcessed_c.shape());
                        assertArrayEquals(shape_rnn, twiceProcessed_f.shape());
                        assertEquals(activationsRnn_c, twiceProcessed_c);
                        assertEquals(activationsRnn_c, twiceProcessed_f);
                        //Second way to check: compare to ComposableInputPreProcessor(RNNtoFF, FFtoCNN)
                        InputPreProcessor compProc = new ComposableInputPreProcessor(new RnnToFeedForwardPreProcessor(), new FeedForwardToCnnPreProcessor(inputHeight, inputWidth, nChannels));
                        INDArray activationsCnnComp_c = compProc.preProcess(activationsRnn_c, miniBatchSize);
                        INDArray activationsCnnComp_f = compProc.preProcess(activationsRnn_f, miniBatchSize);
                        assertEquals(activationsCnnComp_c, activationsCnn_c);
                        assertEquals(activationsCnnComp_f, activationsCnn_f);
                        int[] epsilonShape = new int[] { miniBatchSize * timeSeriesLength, nChannels, inputHeight, inputWidth };
                        rand = Nd4j.rand(epsilonShape);
                        INDArray epsilonsCnn_c = Nd4j.create(epsilonShape, 'c');
                        INDArray epsilonsCnn_f = Nd4j.create(epsilonShape, 'f');
                        epsilonsCnn_c.assign(rand);
                        epsilonsCnn_f.assign(rand);
                        INDArray epsilonsRnnComp_c = compProc.backprop(epsilonsCnn_c, miniBatchSize);
                        INDArray epsilonsRnnComp_f = compProc.backprop(epsilonsCnn_f, miniBatchSize);
                        assertEquals(epsilonsRnnComp_c, epsilonsRnnComp_f);
                        INDArray epsilonsRnn_c = proc.backprop(epsilonsCnn_c, miniBatchSize);
                        INDArray epsilonsRnn_f = proc.backprop(epsilonsCnn_f, miniBatchSize);
                        assertEquals(epsilonsRnn_c, epsilonsRnn_f);
                        if (!epsilonsRnn_c.equals(epsilonsRnnComp_c)) {
                            System.out.println(miniBatchSize + "\t" + timeSeriesLength + "\t" + inputHeight + "\t" + inputWidth + "\t" + nChannels);
                            System.out.println("expected - epsilonsRnnComp");
                            System.out.println(Arrays.toString(epsilonsRnnComp_c.shape()));
                            System.out.println(epsilonsRnnComp_c);
                            System.out.println("actual - epsilonsRnn");
                            System.out.println(Arrays.toString(epsilonsRnn_c.shape()));
                            System.out.println(epsilonsRnn_c);
                        }
                        assertEquals(epsilonsRnnComp_c, epsilonsRnn_c);
                        assertEquals(epsilonsRnnComp_c, epsilonsRnn_f);
                    }
                }
            }
        }
    }
}

Example 3 with InputPreProcessor

use of org.deeplearning4j.nn.conf.InputPreProcessor in project deeplearning4j by deeplearning4j.

the class TestPreProcessors method testCnnToRnnPreProcessor.

@Test
public void testCnnToRnnPreProcessor() {
    //Two ways to test this:
    // (a) check that doing preProcess + backprop on a given input gives same result
    // (b) compare to ComposableInputPreProcessor(CNNtoFF, FFtoRNN)
    int[] miniBatchSizes = { 5, 1 };
    int[] timeSeriesLengths = { 9, 1 };
    int[] inputHeights = { 10, 30 };
    int[] inputWidths = { 10, 30 };
    int[] numChannels = { 1, 3, 6 };
    int cnnNChannelsIn = 3;
    Nd4j.getRandom().setSeed(12345);
    System.out.println();
    for (int miniBatchSize : miniBatchSizes) {
        for (int timeSeriesLength : timeSeriesLengths) {
            for (int inputHeight : inputHeights) {
                for (int inputWidth : inputWidths) {
                    for (int nChannels : numChannels) {
                        String msg = "miniBatch=" + miniBatchSize + ", tsLength=" + timeSeriesLength + ", h=" + inputHeight + ", w=" + inputWidth + ", ch=" + nChannels;
                        InputPreProcessor proc = new CnnToRnnPreProcessor(inputHeight, inputWidth, nChannels);
                        NeuralNetConfiguration nnc = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(inputWidth, inputHeight).nIn(cnnNChannelsIn).nOut(nChannels).build()).build();
                        int numParams = nnc.getLayer().initializer().numParams(nnc);
                        INDArray params = Nd4j.create(1, numParams);
                        ConvolutionLayer layer = (ConvolutionLayer) nnc.getLayer().instantiate(nnc, null, 0, params, true);
                        layer.setInputMiniBatchSize(miniBatchSize);
                        INDArray activationsCnn = Nd4j.rand(new int[] { miniBatchSize * timeSeriesLength, nChannels, inputHeight, inputWidth });
                        //Check shape of outputs:
                        int prod = nChannels * inputHeight * inputWidth;
                        INDArray activationsRnn = proc.preProcess(activationsCnn, miniBatchSize);
                        assertArrayEquals(msg, new int[] { miniBatchSize, prod, timeSeriesLength }, activationsRnn.shape());
                        //Check backward pass. Given that activations and epsilons have same shape, they should
                        //be opposite operations - i.e., get the same thing back out
                        INDArray twiceProcessed = proc.backprop(activationsRnn, miniBatchSize);
                        assertArrayEquals(msg, activationsCnn.shape(), twiceProcessed.shape());
                        assertEquals(msg, activationsCnn, twiceProcessed);
                        //Second way to check: compare to ComposableInputPreProcessor(CNNtoFF, FFtoRNN)
                        InputPreProcessor compProc = new ComposableInputPreProcessor(new CnnToFeedForwardPreProcessor(inputHeight, inputWidth, nChannels), new FeedForwardToRnnPreProcessor());
                        INDArray activationsRnnComp = compProc.preProcess(activationsCnn, miniBatchSize);
                        assertEquals(msg, activationsRnnComp, activationsRnn);
                        INDArray epsilonsRnn = Nd4j.rand(new int[] { miniBatchSize, nChannels * inputHeight * inputWidth, timeSeriesLength });
                        INDArray epsilonsCnnComp = compProc.backprop(epsilonsRnn, miniBatchSize);
                        INDArray epsilonsCnn = proc.backprop(epsilonsRnn, miniBatchSize);
                        if (!epsilonsCnn.equals(epsilonsCnnComp)) {
                            System.out.println(miniBatchSize + "\t" + timeSeriesLength + "\t" + inputHeight + "\t" + inputWidth + "\t" + nChannels);
                            System.out.println("expected - epsilonsCnnComp");
                            System.out.println(Arrays.toString(epsilonsCnnComp.shape()));
                            System.out.println(epsilonsCnnComp);
                            System.out.println("actual - epsilonsCnn");
                            System.out.println(Arrays.toString(epsilonsCnn.shape()));
                            System.out.println(epsilonsCnn);
                        }
                        assertEquals(msg, epsilonsCnnComp, epsilonsCnn);
                    }
                }
            }
        }
    }
}

Example 4 with InputPreProcessor

use of org.deeplearning4j.nn.conf.InputPreProcessor in project deeplearning4j by deeplearning4j.

the class KerasSequentialModel method getMultiLayerConfiguration.

/**
     * Configure a MultiLayerConfiguration from this Keras Sequential model configuration.
     *
     * @return          MultiLayerConfiguration
     */
public MultiLayerConfiguration getMultiLayerConfiguration() throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    if (!this.className.equals(MODEL_CLASS_NAME_SEQUENTIAL))
        throw new InvalidKerasConfigurationException("Keras model class name " + this.className + " incompatible with MultiLayerNetwork");
    if (this.inputLayerNames.size() != 1)
        throw new InvalidKerasConfigurationException("MultiLayeNetwork expects only 1 input (found " + this.inputLayerNames.size() + ")");
    if (this.outputLayerNames.size() != 1)
        throw new InvalidKerasConfigurationException("MultiLayeNetwork expects only 1 output (found " + this.outputLayerNames.size() + ")");
    NeuralNetConfiguration.Builder modelBuilder = new NeuralNetConfiguration.Builder();
    NeuralNetConfiguration.ListBuilder listBuilder = modelBuilder.list();
    /* Add layers one at a time. */
    KerasLayer prevLayer = null;
    int layerIndex = 0;
    for (KerasLayer layer : this.layersOrdered) {
        if (layer.usesRegularization())
            modelBuilder.setUseRegularization(true);
        if (layer.isLayer()) {
            int nbInbound = layer.getInboundLayerNames().size();
            if (nbInbound != 1)
                throw new InvalidKerasConfigurationException("Layers in MultiLayerConfiguration must have exactly one inbound layer (found " + nbInbound + " for layer " + layer.getLayerName() + ")");
            if (prevLayer != null) {
                InputType[] inputTypes = new InputType[1];
                InputPreProcessor preprocessor = null;
                if (prevLayer.isInputPreProcessor()) {
                    inputTypes[0] = this.outputTypes.get(prevLayer.getInboundLayerNames().get(0));
                    preprocessor = prevLayer.getInputPreprocessor(inputTypes);
                } else {
                    inputTypes[0] = this.outputTypes.get(prevLayer.getLayerName());
                    preprocessor = layer.getInputPreprocessor(inputTypes);
                }
                if (preprocessor != null)
                    listBuilder.inputPreProcessor(layerIndex, preprocessor);
            }
            listBuilder.layer(layerIndex++, layer.getLayer());
            if (this.outputLayerNames.contains(layer.getLayerName()) && !(layer.getLayer() instanceof IOutputLayer))
                log.warn("Model cannot be trained: output layer " + layer.getLayerName() + " is not an IOutputLayer (no loss function specified)");
        } else if (layer.getVertex() != null)
            throw new InvalidKerasConfigurationException("Cannot add vertex to MultiLayerConfiguration (class name " + layer.getClassName() + ", layer name " + layer.getLayerName() + ")");
        prevLayer = layer;
    }
    InputType inputType = this.layersOrdered.get(0).getOutputType();
    if (inputType != null)
        listBuilder.setInputType(inputType);
    /* Whether to use standard backprop (or BPTT) or truncated BPTT. */
    if (this.useTruncatedBPTT && this.truncatedBPTT > 0)
        listBuilder.backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(truncatedBPTT).tBPTTBackwardLength(truncatedBPTT);
    else
        listBuilder.backpropType(BackpropType.Standard);
    return listBuilder.build();
}

Example 5 with InputPreProcessor

use of org.deeplearning4j.nn.conf.InputPreProcessor in project deeplearning4j by deeplearning4j.

the class ComposableInputPreProcessor method feedForwardMaskArray.

@Override
public Pair<INDArray, MaskState> feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) {
    for (InputPreProcessor preproc : inputPreProcessors) {
        Pair<INDArray, MaskState> p = preproc.feedForwardMaskArray(maskArray, currentMaskState, minibatchSize);
        maskArray = p.getFirst();
        currentMaskState = p.getSecond();
    }
    return new Pair<>(maskArray, currentMaskState);
}