use of org.deeplearning4j.nn.weights.WeightInit in project deeplearning4j by deeplearning4j.
the class MultiNeuralNetConfLayerBuilderTest method testNeuralNetConfigAPI.
@Test
public void testNeuralNetConfigAPI() {
LossFunction newLoss = LossFunction.SQUARED_LOSS;
int newNumIn = numIn + 1;
int newNumOut = numOut + 1;
WeightInit newWeight = WeightInit.UNIFORM;
double newDrop = 0.5;
int[] newFS = new int[] { 3, 3 };
int newFD = 7;
int[] newStride = new int[] { 3, 3 };
Convolution.Type newConvType = Convolution.Type.SAME;
PoolingType newPoolType = PoolingType.AVG;
double newCorrupt = 0.5;
double newSparsity = 0.5;
HiddenUnit newHidden = HiddenUnit.BINARY;
VisibleUnit newVisible = VisibleUnit.BINARY;
MultiLayerConfiguration multiConf1 = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(newNumIn).nOut(newNumOut).activation(act).build()).layer(1, new DenseLayer.Builder().nIn(newNumIn + 1).nOut(newNumOut + 1).activation(act).build()).build();
NeuralNetConfiguration firstLayer = multiConf1.getConf(0);
NeuralNetConfiguration secondLayer = multiConf1.getConf(1);
assertFalse(firstLayer.equals(secondLayer));
}
use of org.deeplearning4j.nn.weights.WeightInit in project deeplearning4j by deeplearning4j.
the class KerasLayer method getWeightInitFromConfig.
/**
* Get weight initialization from Keras layer configuration.
*
* @param layerConfig dictionary containing Keras layer configuration
* @param enforceTrainingConfig
* @return
* @throws InvalidKerasConfigurationException
* @throws UnsupportedKerasConfigurationException
*/
protected WeightInit getWeightInitFromConfig(Map<String, Object> layerConfig, boolean enforceTrainingConfig) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
Map<String, Object> innerConfig = getInnerLayerConfigFromConfig(layerConfig);
if (!innerConfig.containsKey(LAYER_FIELD_INIT))
throw new InvalidKerasConfigurationException("Keras layer is missing " + LAYER_FIELD_INIT + " field");
String kerasInit = (String) innerConfig.get(LAYER_FIELD_INIT);
WeightInit init;
try {
init = mapWeightInitialization(kerasInit);
} catch (UnsupportedKerasConfigurationException e) {
if (enforceTrainingConfig)
throw e;
else {
init = WeightInit.XAVIER;
log.warn("Unknown weight initializer " + kerasInit + " (Using XAVIER instead).");
}
}
return init;
}
use of org.deeplearning4j.nn.weights.WeightInit in project deeplearning4j by deeplearning4j.
the class KerasLstm method getRecurrentWeightInitFromConfig.
/**
* Get LSTM recurrent weight initialization from Keras layer configuration.
*
* @param layerConfig dictionary containing Keras layer configuration
* @return epsilon
* @throws InvalidKerasConfigurationException
*/
public static WeightInit getRecurrentWeightInitFromConfig(Map<String, Object> layerConfig, boolean train) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
Map<String, Object> innerConfig = getInnerLayerConfigFromConfig(layerConfig);
if (!innerConfig.containsKey(LAYER_FIELD_INNER_INIT))
throw new InvalidKerasConfigurationException("Keras LSTM layer config missing " + LAYER_FIELD_INNER_INIT + " field");
String kerasInit = (String) innerConfig.get(LAYER_FIELD_INNER_INIT);
WeightInit init;
try {
init = mapWeightInitialization(kerasInit);
} catch (UnsupportedKerasConfigurationException e) {
if (train)
throw e;
else {
init = WeightInit.XAVIER;
log.warn("Unknown weight initializer " + kerasInit + " (Using XAVIER instead).");
}
}
return init;
}
use of org.deeplearning4j.nn.weights.WeightInit in project deeplearning4j by deeplearning4j.
the class TrainModule method getLayerInfoTable.
private String[][] getLayerInfoTable(int layerIdx, TrainModuleUtils.GraphInfo gi, I18N i18N, boolean noData, StatsStorage ss, String wid) {
List<String[]> layerInfoRows = new ArrayList<>();
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerName"), gi.getVertexNames().get(layerIdx) });
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerType"), "" });
if (!noData) {
Persistable p = ss.getStaticInfo(currentSessionID, StatsListener.TYPE_ID, wid);
if (p != null) {
StatsInitializationReport initReport = (StatsInitializationReport) p;
String configJson = initReport.getModelConfigJson();
String modelClass = initReport.getModelClassName();
//TODO error handling...
String layerType = "";
Layer layer = null;
NeuralNetConfiguration nnc = null;
if (modelClass.endsWith("MultiLayerNetwork")) {
MultiLayerConfiguration conf = MultiLayerConfiguration.fromJson(configJson);
//-1 because of input
int confIdx = layerIdx - 1;
if (confIdx >= 0) {
nnc = conf.getConf(confIdx);
layer = nnc.getLayer();
} else {
//Input layer
layerType = "Input";
}
} else if (modelClass.endsWith("ComputationGraph")) {
ComputationGraphConfiguration conf = ComputationGraphConfiguration.fromJson(configJson);
String vertexName = gi.getVertexNames().get(layerIdx);
Map<String, GraphVertex> vertices = conf.getVertices();
if (vertices.containsKey(vertexName) && vertices.get(vertexName) instanceof LayerVertex) {
LayerVertex lv = (LayerVertex) vertices.get(vertexName);
nnc = lv.getLayerConf();
layer = nnc.getLayer();
} else if (conf.getNetworkInputs().contains(vertexName)) {
layerType = "Input";
} else {
GraphVertex gv = conf.getVertices().get(vertexName);
if (gv != null) {
layerType = gv.getClass().getSimpleName();
}
}
} else if (modelClass.endsWith("VariationalAutoencoder")) {
layerType = gi.getVertexTypes().get(layerIdx);
Map<String, String> map = gi.getVertexInfo().get(layerIdx);
for (Map.Entry<String, String> entry : map.entrySet()) {
layerInfoRows.add(new String[] { entry.getKey(), entry.getValue() });
}
}
if (layer != null) {
layerType = getLayerType(layer);
}
if (layer != null) {
String activationFn = null;
if (layer instanceof FeedForwardLayer) {
FeedForwardLayer ffl = (FeedForwardLayer) layer;
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerNIn"), String.valueOf(ffl.getNIn()) });
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerSize"), String.valueOf(ffl.getNOut()) });
activationFn = layer.getActivationFn().toString();
}
int nParams = layer.initializer().numParams(nnc);
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerNParams"), String.valueOf(nParams) });
if (nParams > 0) {
WeightInit wi = layer.getWeightInit();
String str = wi.toString();
if (wi == WeightInit.DISTRIBUTION) {
str += layer.getDist();
}
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerWeightInit"), str });
Updater u = layer.getUpdater();
String us = (u == null ? "" : u.toString());
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerUpdater"), us });
//TODO: Maybe L1/L2, dropout, updater-specific values etc
}
if (layer instanceof ConvolutionLayer || layer instanceof SubsamplingLayer) {
int[] kernel;
int[] stride;
int[] padding;
if (layer instanceof ConvolutionLayer) {
ConvolutionLayer cl = (ConvolutionLayer) layer;
kernel = cl.getKernelSize();
stride = cl.getStride();
padding = cl.getPadding();
} else {
SubsamplingLayer ssl = (SubsamplingLayer) layer;
kernel = ssl.getKernelSize();
stride = ssl.getStride();
padding = ssl.getPadding();
activationFn = null;
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerSubsamplingPoolingType"), ssl.getPoolingType().toString() });
}
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerCnnKernel"), Arrays.toString(kernel) });
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerCnnStride"), Arrays.toString(stride) });
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerCnnPadding"), Arrays.toString(padding) });
}
if (activationFn != null) {
layerInfoRows.add(new String[] { i18N.getMessage("train.model.layerinfotable.layerActivationFn"), activationFn });
}
}
layerInfoRows.get(1)[1] = layerType;
}
}
return layerInfoRows.toArray(new String[layerInfoRows.size()][0]);
}
use of org.deeplearning4j.nn.weights.WeightInit in project deeplearning4j by deeplearning4j.
the class FineTuneConfiguration method applyToNeuralNetConfiguration.
public void applyToNeuralNetConfiguration(NeuralNetConfiguration nnc) {
Layer l = nnc.getLayer();
Updater originalUpdater = null;
WeightInit origWeightInit = null;
if (l != null) {
originalUpdater = l.getUpdater();
origWeightInit = l.getWeightInit();
if (activationFn != null)
l.setActivationFn(activationFn);
if (weightInit != null)
l.setWeightInit(weightInit);
if (biasInit != null)
l.setBiasInit(biasInit);
if (dist != null)
l.setDist(dist);
if (learningRate != null) {
//usually the same learning rate is applied to both bias and weights
//so always overwrite the learning rate to both?
l.setLearningRate(learningRate);
l.setBiasLearningRate(learningRate);
}
if (biasLearningRate != null)
l.setBiasLearningRate(biasLearningRate);
if (learningRateSchedule != null)
l.setLearningRateSchedule(learningRateSchedule);
// if(lrScoreBasedDecay != null)
if (l1 != null)
l.setL1(l1);
if (l2 != null)
l.setL2(l2);
if (l1Bias != null)
l.setL1Bias(l1Bias);
if (l2Bias != null)
l.setL2Bias(l2Bias);
if (dropOut != null)
l.setDropOut(dropOut);
if (updater != null)
l.setUpdater(updater);
if (momentum != null)
l.setMomentum(momentum);
if (momentumSchedule != null)
l.setMomentum(momentum);
if (epsilon != null)
l.setEpsilon(epsilon);
if (rho != null)
l.setRho(rho);
if (rmsDecay != null)
l.setRmsDecay(rmsDecay);
if (adamMeanDecay != null)
l.setAdamMeanDecay(adamMeanDecay);
if (adamVarDecay != null)
l.setAdamVarDecay(adamVarDecay);
}
if (miniBatch != null)
nnc.setMiniBatch(miniBatch);
if (numIterations != null)
nnc.setNumIterations(numIterations);
if (maxNumLineSearchIterations != null)
nnc.setMaxNumLineSearchIterations(maxNumLineSearchIterations);
if (seed != null)
nnc.setSeed(seed);
if (useRegularization != null)
nnc.setUseRegularization(useRegularization);
if (optimizationAlgo != null)
nnc.setOptimizationAlgo(optimizationAlgo);
if (stepFunction != null)
nnc.setStepFunction(stepFunction);
if (useDropConnect != null)
nnc.setUseDropConnect(useDropConnect);
if (minimize != null)
nnc.setMinimize(minimize);
if (gradientNormalization != null)
l.setGradientNormalization(gradientNormalization);
if (gradientNormalizationThreshold != null)
l.setGradientNormalizationThreshold(gradientNormalizationThreshold);
if (learningRatePolicy != null)
nnc.setLearningRatePolicy(learningRatePolicy);
if (lrPolicySteps != null)
nnc.setLrPolicySteps(lrPolicySteps);
if (lrPolicyPower != null)
nnc.setLrPolicyPower(lrPolicyPower);
if (convolutionMode != null && l instanceof ConvolutionLayer) {
((ConvolutionLayer) l).setConvolutionMode(convolutionMode);
}
if (convolutionMode != null && l instanceof SubsamplingLayer) {
((SubsamplingLayer) l).setConvolutionMode(convolutionMode);
}
//Check the updater config. If we change updaters, we want to remove the old config to avoid warnings
if (l != null && updater != null && originalUpdater != null && updater != originalUpdater) {
switch(originalUpdater) {
case ADAM:
if (adamMeanDecay == null)
l.setAdamMeanDecay(Double.NaN);
if (adamVarDecay == null)
l.setAdamVarDecay(Double.NaN);
break;
case ADADELTA:
if (rho == null)
l.setRho(Double.NaN);
if (epsilon == null)
l.setEpsilon(Double.NaN);
break;
case NESTEROVS:
if (momentum == null)
l.setMomentum(Double.NaN);
if (momentumSchedule == null)
l.setMomentumSchedule(null);
if (epsilon == null)
l.setEpsilon(Double.NaN);
break;
case ADAGRAD:
if (epsilon == null)
l.setEpsilon(Double.NaN);
break;
case RMSPROP:
if (rmsDecay == null)
l.setRmsDecay(Double.NaN);
if (epsilon == null)
l.setEpsilon(Double.NaN);
break;
}
}
//Check weight init. Remove dist if originally was DISTRIBUTION, and isn't now -> remove no longer needed distribution
if (l != null && origWeightInit == WeightInit.DISTRIBUTION && weightInit != null && weightInit != WeightInit.DISTRIBUTION) {
l.setDist(null);
}
//Perform validation. This also sets the defaults for updaters. For example, Updater.RMSProp -> set rmsDecay
if (l != null) {
LayerValidation.updaterValidation(l.getLayerName(), l, momentum, momentumSchedule, adamMeanDecay, adamVarDecay, rho, rmsDecay, epsilon);
boolean useDropCon = (useDropConnect == null ? nnc.isUseDropConnect() : useDropConnect);
LayerValidation.generalValidation(l.getLayerName(), l, nnc.isUseRegularization(), useDropCon, dropOut, l2, l2Bias, l1, l1Bias, dist);
}
//Also: update the LR, L1 and L2 maps, based on current config (which might be different to original config)
if (nnc.variables(false) != null) {
for (String s : nnc.variables(false)) {
nnc.setLayerParamLR(s);
}
}
}
Aggregations