Examples with NeuralNetConfiguration org.deeplearning4j.nn.conf.NeuralNetConfiguration used on opensource projects

Example 31 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestGradientNormalization method testL2ClippingPerLayer.

@Test
public void testL2ClippingPerLayer() {
    Nd4j.getRandom().setSeed(12345);
    double threshold = 3;
    for (int t = 0; t < 2; t++) {
        //t=0: small -> no clipping
        //t=1: large -> clipping
        NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new DenseLayer.Builder().nIn(10).nOut(20).updater(org.deeplearning4j.nn.conf.Updater.NONE).gradientNormalization(GradientNormalization.ClipL2PerLayer).gradientNormalizationThreshold(threshold).build()).build();
        int numParams = conf.getLayer().initializer().numParams(conf);
        INDArray params = Nd4j.create(1, numParams);
        Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);
        Updater updater = UpdaterCreator.getUpdater(layer);
        INDArray weightGrad = Nd4j.rand(10, 20).muli((t == 0 ? 0.05 : 10));
        INDArray biasGrad = Nd4j.rand(1, 10).muli((t == 0 ? 0.05 : 10));
        INDArray weightGradCopy = weightGrad.dup();
        INDArray biasGradCopy = biasGrad.dup();
        Gradient gradient = new DefaultGradient();
        gradient.setGradientFor(DefaultParamInitializer.WEIGHT_KEY, weightGrad);
        gradient.setGradientFor(DefaultParamInitializer.BIAS_KEY, biasGrad);
        double layerGradL2 = gradient.gradient().norm2Number().doubleValue();
        if (t == 0)
            assertTrue(layerGradL2 < threshold);
        else
            assertTrue(layerGradL2 > threshold);
        updater.update(layer, gradient, 0, 1);
        if (t == 0) {
            //norm2 < threshold -> no change
            assertEquals(weightGradCopy, weightGrad);
            assertEquals(biasGradCopy, biasGrad);
            continue;
        } else {
            //norm2 > threshold -> rescale
            assertNotEquals(weightGradCopy, weightGrad);
            assertNotEquals(biasGradCopy, biasGrad);
        }
        //for above threshold only...
        double scalingFactor = threshold / layerGradL2;
        INDArray expectedWeightGrad = weightGradCopy.mul(scalingFactor);
        INDArray expectedBiasGrad = biasGradCopy.mul(scalingFactor);
        assertEquals(expectedWeightGrad, gradient.getGradientFor(DefaultParamInitializer.WEIGHT_KEY));
        assertEquals(expectedBiasGrad, gradient.getGradientFor(DefaultParamInitializer.BIAS_KEY));
    }
}

Example 32 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TestUpdaters method testNoOpUpdater.

@Test
public void testNoOpUpdater() {
    Random r = new Random(12345L);
    double lr = 0.5;
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr).layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).updater(org.deeplearning4j.nn.conf.Updater.NONE).build()).build();
    int numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);
    Updater updater = UpdaterCreator.getUpdater(layer);
    for (int i = 0; i < weightGradient.length(); i++) weightGradient.putScalar(i, r.nextDouble());
    for (int i = 0; i < biasGradient.length(); i++) biasGradient.putScalar(i, r.nextDouble());
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradient);
    gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradient);
    updater.update(layer, gradient, -1, 1);
    INDArray weightGradActual = gradient.getGradientFor(DefaultParamInitializer.WEIGHT_KEY);
    INDArray biasGradActual = gradient.getGradientFor(DefaultParamInitializer.BIAS_KEY);
    assertEquals(weightGradient, weightGradActual);
    assertEquals(biasGradient, biasGradActual);
}

Example 33 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class BaseLayer method transpose.

@Override
public Layer transpose() {
    if (!(conf.getLayer() instanceof org.deeplearning4j.nn.conf.layers.FeedForwardLayer))
        throw new UnsupportedOperationException("unsupported layer type: " + conf.getLayer().getClass().getName());
    INDArray w = getParam(DefaultParamInitializer.WEIGHT_KEY);
    INDArray b = getParam(DefaultParamInitializer.BIAS_KEY);
    INDArray vb = getParam(PretrainParamInitializer.VISIBLE_BIAS_KEY);
    Layer layer;
    try {
        // assume a deep clone here
        NeuralNetConfiguration clone = conf.clone();
        org.deeplearning4j.nn.conf.layers.FeedForwardLayer clonedLayerConf = (org.deeplearning4j.nn.conf.layers.FeedForwardLayer) clone.getLayer();
        int nIn = clonedLayerConf.getNOut();
        int nOut = clonedLayerConf.getNIn();
        clonedLayerConf.setNIn(nIn);
        clonedLayerConf.setNOut(nOut);
        //Need to swap the hidden and visible biases for pretrain layers
        INDArray newB;
        INDArray newVB = null;
        int totalParams = w.length();
        if (vb != null) {
            newB = vb.dup();
            newVB = b.dup();
            totalParams += newB.length() + newVB.length();
        } else {
            newB = Nd4j.create(1, nOut);
            totalParams += newB.length();
        }
        INDArray paramsView = Nd4j.create(1, totalParams);
        layer = clone.getLayer().instantiate(clone, iterationListeners, this.index, paramsView, true);
        layer.setParam(DefaultParamInitializer.WEIGHT_KEY, w.transpose().dup());
        layer.setParam(DefaultParamInitializer.BIAS_KEY, newB);
        if (vb != null)
            layer.setParam(PretrainParamInitializer.VISIBLE_BIAS_KEY, newVB);
    } catch (Exception e) {
        throw new RuntimeException("unable to construct transposed layer", e);
    }
    return layer;
}

Example 34 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class MultiLayerNetwork method printConfiguration.

/**
     * Prints the configuration
     */
public void printConfiguration() {
    StringBuilder sb = new StringBuilder();
    int count = 0;
    for (NeuralNetConfiguration conf : getLayerWiseConfigurations().getConfs()) {
        sb.append(" Layer " + count++ + " conf " + conf);
    }
    log.info(sb.toString());
}

Example 35 with NeuralNetConfiguration

use of org.deeplearning4j.nn.conf.NeuralNetConfiguration in project deeplearning4j by deeplearning4j.

the class TrainModule method getConfig.

private Triple<MultiLayerConfiguration, ComputationGraphConfiguration, NeuralNetConfiguration> getConfig() {
    boolean noData = currentSessionID == null;
    StatsStorage ss = (noData ? null : knownSessionIDs.get(currentSessionID));
    List<Persistable> allStatic = (noData ? Collections.EMPTY_LIST : ss.getAllStaticInfos(currentSessionID, StatsListener.TYPE_ID));
    if (allStatic.size() == 0)
        return null;
    StatsInitializationReport p = (StatsInitializationReport) allStatic.get(0);
    String modelClass = p.getModelClassName();
    String config = p.getModelConfigJson();
    if (modelClass.endsWith("MultiLayerNetwork")) {
        MultiLayerConfiguration conf = MultiLayerConfiguration.fromJson(config);
        return new Triple<>(conf, null, null);
    } else if (modelClass.endsWith("ComputationGraph")) {
        ComputationGraphConfiguration conf = ComputationGraphConfiguration.fromJson(config);
        return new Triple<>(null, conf, null);
    } else {
        try {
            NeuralNetConfiguration layer = NeuralNetConfiguration.mapper().readValue(config, NeuralNetConfiguration.class);
            return new Triple<>(null, null, layer);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return null;
}