use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.
the class VaeGradientCheckTests method testVaeAsMLP.
@Test
public void testVaeAsMLP() {
//Post pre-training: a VAE can be used as a MLP, by taking the mean value from p(z|x) as the output
//This gradient check tests this part
//activation functions such as relu and hardtanh: may randomly fail due to discontinuities
String[] activFns = { "identity", "tanh" };
LossFunction[] lossFunctions = { LossFunction.MCXENT, LossFunction.MSE };
//i.e., lossFunctions[i] used with outputActivations[i] here
String[] outputActivations = { "softmax", "tanh" };
//use l2vals[i] with l1vals[i]
double[] l2vals = { 0.4, 0.0, 0.4, 0.4 };
double[] l1vals = { 0.0, 0.0, 0.5, 0.0 };
double[] biasL2 = { 0.0, 0.0, 0.0, 0.2 };
double[] biasL1 = { 0.0, 0.0, 0.6, 0.0 };
int[][] encoderLayerSizes = new int[][] { { 5 }, { 5, 6 } };
int[][] decoderLayerSizes = new int[][] { { 6 }, { 7, 8 } };
Nd4j.getRandom().setSeed(12345);
for (int minibatch : new int[] { 1, 5 }) {
INDArray input = Nd4j.rand(minibatch, 4);
INDArray labels = Nd4j.create(minibatch, 3);
for (int i = 0; i < minibatch; i++) {
labels.putScalar(i, i % 3, 1.0);
}
for (int ls = 0; ls < encoderLayerSizes.length; ls++) {
int[] encoderSizes = encoderLayerSizes[ls];
int[] decoderSizes = decoderLayerSizes[ls];
for (String afn : activFns) {
for (int i = 0; i < lossFunctions.length; i++) {
for (int k = 0; k < l2vals.length; k++) {
LossFunction lf = lossFunctions[i];
String outputActivation = outputActivations[i];
double l2 = l2vals[k];
double l1 = l1vals[k];
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(true).l2(l2).l1(l1).l2Bias(biasL2[k]).l1Bias(biasL1[k]).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(1.0).seed(12345L).list().layer(0, new VariationalAutoencoder.Builder().nIn(4).nOut(3).encoderLayerSizes(encoderSizes).decoderLayerSizes(decoderSizes).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).activation(afn).updater(Updater.SGD).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nIn(3).nOut(3).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.SGD).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
String msg = "testVaeAsMLP() - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", encLayerSizes = " + Arrays.toString(encoderSizes) + ", decLayerSizes = " + Arrays.toString(decoderSizes) + ", l2=" + l2 + ", l1=" + l1;
if (PRINT_RESULTS) {
System.out.println(msg);
for (int j = 0; j < mln.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
}
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
assertTrue(msg, gradOK);
}
}
}
}
}
}
use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.
the class MultiLayerNeuralNetConfigurationTest method testListOverloads.
@Test
public void testListOverloads() {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(1, new OutputLayer.Builder().nIn(4).nOut(5).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
DenseLayer dl = (DenseLayer) conf.getConf(0).getLayer();
assertEquals(3, dl.getNIn());
assertEquals(4, dl.getNOut());
OutputLayer ol = (OutputLayer) conf.getConf(1).getLayer();
assertEquals(4, ol.getNIn());
assertEquals(5, ol.getNOut());
MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(1, new OutputLayer.Builder().nIn(4).nOut(5).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
net2.init();
MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345).list(new DenseLayer.Builder().nIn(3).nOut(4).build(), new OutputLayer.Builder().nIn(4).nOut(5).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net3 = new MultiLayerNetwork(conf3);
net3.init();
assertEquals(conf, conf2);
assertEquals(conf, conf3);
}
use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.
the class MultiLayerNeuralNetConfigurationTest method testInvalidConfig.
@Test
public void testInvalidConfig() {
try {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
fail("No exception thrown for invalid configuration");
} catch (IllegalStateException e) {
//OK
e.printStackTrace();
} catch (Throwable e) {
e.printStackTrace();
fail("Unexpected exception thrown for invalid config");
}
try {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(1, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
fail("No exception thrown for invalid configuration");
} catch (IllegalStateException e) {
//OK
e.printStackTrace();
} catch (Throwable e) {
e.printStackTrace();
fail("Unexpected exception thrown for invalid config");
}
try {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.init();
fail("No exception thrown for invalid configuration");
} catch (IllegalStateException e) {
//OK
e.printStackTrace();
} catch (Throwable e) {
e.printStackTrace();
fail("Unexpected exception thrown for invalid config");
}
}
use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.
the class MultiNeuralNetConfLayerBuilderTest method testRbmSetup.
@Test
public void testRbmSetup() throws Exception {
MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).seed(123).iterations(5).maxNumLineSearchIterations(// Magical Optimisation Stuff
10).regularization(true).list().layer(0, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(784).nOut(1000).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(1000).nOut(500).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(2, new RBM.Builder(RBM.HiddenUnit.RECTIFIED, RBM.VisibleUnit.GAUSSIAN).nIn(500).nOut(250).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(250).nOut(10).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork network = new MultiLayerNetwork(multiLayerConfiguration);
network.init();
DataSet d = new MnistDataSetIterator(2, 2).next();
org.deeplearning4j.nn.api.Layer firstRbm = network.getLayer(0);
org.deeplearning4j.nn.api.Layer secondRbm = network.getLayer(1);
org.deeplearning4j.nn.api.Layer thirdRbm = network.getLayer(2);
org.deeplearning4j.nn.api.Layer fourthRbm = network.getLayer(3);
INDArray[] weightMatrices = new INDArray[] { firstRbm.getParam(DefaultParamInitializer.WEIGHT_KEY), secondRbm.getParam(DefaultParamInitializer.WEIGHT_KEY), thirdRbm.getParam(DefaultParamInitializer.WEIGHT_KEY), fourthRbm.getParam(DefaultParamInitializer.WEIGHT_KEY) };
INDArray[] hiddenBiases = new INDArray[] { firstRbm.getParam(DefaultParamInitializer.BIAS_KEY), secondRbm.getParam(DefaultParamInitializer.BIAS_KEY), thirdRbm.getParam(DefaultParamInitializer.BIAS_KEY), fourthRbm.getParam(DefaultParamInitializer.BIAS_KEY) };
int[][] shapeAssertions = new int[][] { { 784, 1000 }, { 1000, 500 }, { 500, 250 }, { 250, 10 } };
int[][] biasAssertions = new int[][] { { 1, 1000 }, { 1, 500 }, { 1, 250 }, { 1, 10 } };
for (int i = 0; i < shapeAssertions.length; i++) {
assertArrayEquals(shapeAssertions[i], weightMatrices[i].shape());
assertArrayEquals(biasAssertions[i], hiddenBiases[i].shape());
}
network.fit(d);
}
use of org.deeplearning4j.nn.multilayer.MultiLayerNetwork in project deeplearning4j by deeplearning4j.
the class GradientCheckTests method testGradientGravesBidirectionalLSTMEdgeCases.
@Test
public void testGradientGravesBidirectionalLSTMEdgeCases() {
//Edge cases: T=1, miniBatchSize=1, both
int[] timeSeriesLength = { 1, 5, 1 };
int[] miniBatchSize = { 7, 1, 1 };
int nIn = 7;
int layerSize = 9;
int nOut = 4;
for (int i = 0; i < timeSeriesLength.length; i++) {
Random r = new Random(12345L);
INDArray input = Nd4j.zeros(miniBatchSize[i], nIn, timeSeriesLength[i]);
for (int m = 0; m < miniBatchSize[i]; m++) {
for (int j = 0; j < nIn; j++) {
for (int k = 0; k < timeSeriesLength[i]; k++) {
input.putScalar(new int[] { m, j, k }, r.nextDouble() - 0.5);
}
}
}
INDArray labels = Nd4j.zeros(miniBatchSize[i], nOut, timeSeriesLength[i]);
for (int m = 0; m < miniBatchSize[i]; m++) {
for (int j = 0; j < timeSeriesLength[i]; j++) {
int idx = r.nextInt(nOut);
labels.putScalar(new int[] { m, idx, j }, 1.0f);
}
}
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).seed(12345L).list().layer(0, new GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(layerSize).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).layer(1, new RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).weightInit(WeightInit.DISTRIBUTION).dist(new NormalDistribution(0, 1)).updater(Updater.NONE).build()).pretrain(false).backprop(true).build();
MultiLayerNetwork mln = new MultiLayerNetwork(conf);
mln.init();
boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
String msg = "testGradientGravesLSTMEdgeCases() - timeSeriesLength=" + timeSeriesLength[i] + ", miniBatchSize=" + miniBatchSize[i];
assertTrue(msg, gradOK);
}
}
Aggregations