use of ml.shifu.shifu.core.dtrain.gs.GridSearch in project shifu by ShifuML.
the class MetaFactory method validate.
/**
* Validate the ModelConfig object, to make sure each item follow the constrain
*
* @param modelConfig
* - object to validate
* @return ValidateResult
* If all items are OK, the ValidateResult.status will be true;
* Or the ValidateResult.status will be false, ValidateResult.causes will contain the reasons
* @throws Exception
* any exception in validaiton
*/
public static ValidateResult validate(ModelConfig modelConfig) throws Exception {
ValidateResult result = new ValidateResult(true);
GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
Class<?> cls = modelConfig.getClass();
Field[] fields = cls.getDeclaredFields();
for (Field field : fields) {
// skip log instance
if (field.getName().equalsIgnoreCase("log") || field.getName().equalsIgnoreCase("logger")) {
continue;
}
if (!field.isSynthetic()) {
Method method = cls.getMethod("get" + getMethodName(field.getName()));
Object value = method.invoke(modelConfig);
if (value instanceof List) {
List<?> objList = (List<?>) value;
for (Object obj : objList) {
encapsulateResult(result, iterateCheck(gs.hasHyperParam(), field.getName(), obj));
}
} else {
encapsulateResult(result, iterateCheck(gs.hasHyperParam(), field.getName(), value));
}
}
}
return result;
}
use of ml.shifu.shifu.core.dtrain.gs.GridSearch in project shifu by ShifuML.
the class AbstractNNWorker method init.
@Override
public void init(WorkerContext<NNParams, NNParams> context) {
// load props firstly
this.props = context.getProps();
loadConfigFiles(context.getProps());
this.trainerId = Integer.valueOf(context.getProps().getProperty(CommonConstants.SHIFU_TRAINER_ID, "0"));
GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
this.validParams = this.modelConfig.getTrain().getParams();
if (gs.hasHyperParam()) {
this.validParams = gs.getParams(trainerId);
LOG.info("Start grid search master with params: {}", validParams);
}
Integer kCrossValidation = this.modelConfig.getTrain().getNumKFold();
if (kCrossValidation != null && kCrossValidation > 0) {
isKFoldCV = true;
LOG.info("Cross validation is enabled by kCrossValidation: {}.", kCrossValidation);
}
this.poissonSampler = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(NNConstants.NN_POISON_SAMPLER));
this.rng = new PoissonDistribution(1.0d);
Double upSampleWeight = modelConfig.getTrain().getUpSampleWeight();
if (Double.compare(upSampleWeight, 1d) != 0 && (modelConfig.isRegression() || (modelConfig.isClassification() && modelConfig.getTrain().isOneVsAll()))) {
// set mean to upSampleWeight -1 and get sample + 1to make sure no zero sample value
LOG.info("Enable up sampling with weight {}.", upSampleWeight);
this.upSampleRng = new PoissonDistribution(upSampleWeight - 1);
}
Integer epochsPerIterationInteger = this.modelConfig.getTrain().getEpochsPerIteration();
this.epochsPerIteration = epochsPerIterationInteger == null ? 1 : epochsPerIterationInteger.intValue();
LOG.info("epochsPerIteration in worker is :{}", epochsPerIteration);
// Object elmObject = validParams.get(DTrainUtils.IS_ELM);
// isELM = elmObject == null ? false : "true".equalsIgnoreCase(elmObject.toString());
// LOG.info("Check isELM: {}", isELM);
Object dropoutRateObj = validParams.get(CommonConstants.DROPOUT_RATE);
if (dropoutRateObj != null) {
this.dropoutRate = Double.valueOf(dropoutRateObj.toString());
}
LOG.info("'dropoutRate' in worker is :{}", this.dropoutRate);
Object miniBatchO = validParams.get(CommonConstants.MINI_BATCH);
if (miniBatchO != null) {
int miniBatchs;
try {
miniBatchs = Integer.parseInt(miniBatchO.toString());
} catch (Exception e) {
miniBatchs = 1;
}
if (miniBatchs < 0) {
this.batchs = 1;
} else if (miniBatchs > 1000) {
this.batchs = 1000;
} else {
this.batchs = miniBatchs;
}
LOG.info("'miniBatchs' in worker is : {}, batchs is {} ", miniBatchs, batchs);
}
int[] inputOutputIndex = DTrainUtils.getInputOutputCandidateCounts(modelConfig.getNormalizeType(), this.columnConfigList);
this.inputNodeCount = inputOutputIndex[0] == 0 ? inputOutputIndex[2] : inputOutputIndex[0];
// if is one vs all classification, outputNodeCount is set to 1, if classes=2, outputNodeCount is also 1
int classes = modelConfig.getTags().size();
this.outputNodeCount = (isLinearTarget || modelConfig.isRegression()) ? inputOutputIndex[1] : (modelConfig.getTrain().isOneVsAll() ? inputOutputIndex[1] : (classes == 2 ? 1 : classes));
this.candidateCount = inputOutputIndex[2];
boolean isAfterVarSelect = inputOutputIndex[0] != 0;
LOG.info("isAfterVarSelect {}: Input count {}, output count {}, candidate count {}", isAfterVarSelect, inputNodeCount, outputNodeCount, candidateCount);
// cache all feature list for sampling features
this.allFeatures = NormalUtils.getAllFeatureList(columnConfigList, isAfterVarSelect);
String subsetStr = context.getProps().getProperty(CommonConstants.SHIFU_NN_FEATURE_SUBSET);
if (StringUtils.isBlank(subsetStr)) {
this.subFeatures = this.allFeatures;
} else {
String[] splits = subsetStr.split(",");
this.subFeatures = new ArrayList<Integer>(splits.length);
for (String split : splits) {
int featureIndex = Integer.parseInt(split);
this.subFeatures.add(featureIndex);
}
}
this.subFeatureSet = new HashSet<Integer>(this.subFeatures);
LOG.info("subFeatures size is {}", subFeatures.size());
this.featureInputsCnt = DTrainUtils.getFeatureInputsCnt(this.modelConfig, this.columnConfigList, this.subFeatureSet);
this.wgtInit = "default";
Object wgtInitObj = validParams.get(CommonConstants.WEIGHT_INITIALIZER);
if (wgtInitObj != null) {
this.wgtInit = wgtInitObj.toString();
}
Object lossObj = validParams.get("Loss");
this.lossStr = lossObj != null ? lossObj.toString() : "squared";
LOG.info("Loss str is {}", this.lossStr);
this.isDry = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(CommonConstants.SHIFU_DRY_DTRAIN));
this.isSpecificValidation = (modelConfig.getValidationDataSetRawPath() != null && !"".equals(modelConfig.getValidationDataSetRawPath()));
this.isStratifiedSampling = this.modelConfig.getTrain().getStratifiedSample();
if (isOnDisk()) {
LOG.info("NNWorker is loading data into disk.");
try {
initDiskDataSet();
} catch (IOException e) {
throw new RuntimeException(e);
}
// cannot find a good place to close these two data set, using Shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@Override
public void run() {
((BufferedFloatMLDataSet) (AbstractNNWorker.this.trainingData)).close();
((BufferedFloatMLDataSet) (AbstractNNWorker.this.validationData)).close();
}
}));
} else {
LOG.info("NNWorker is loading data into memory.");
double memoryFraction = Double.valueOf(context.getProps().getProperty("guagua.data.memoryFraction", "0.6"));
long memoryStoreSize = (long) (Runtime.getRuntime().maxMemory() * memoryFraction);
LOG.info("Max heap memory: {}, fraction: {}", Runtime.getRuntime().maxMemory(), memoryFraction);
double crossValidationRate = this.modelConfig.getValidSetRate();
try {
if (StringUtils.isNotBlank(modelConfig.getValidationDataSetRawPath())) {
// fixed 0.6 and 0.4 of max memory for trainingData and validationData
this.trainingData = new MemoryDiskFloatMLDataSet((long) (memoryStoreSize * 0.6), DTrainUtils.getTrainingFile().toString(), this.featureInputsCnt, this.outputNodeCount);
this.validationData = new MemoryDiskFloatMLDataSet((long) (memoryStoreSize * 0.4), DTrainUtils.getTestingFile().toString(), this.featureInputsCnt, this.outputNodeCount);
} else {
this.trainingData = new MemoryDiskFloatMLDataSet((long) (memoryStoreSize * (1 - crossValidationRate)), DTrainUtils.getTrainingFile().toString(), this.featureInputsCnt, this.outputNodeCount);
this.validationData = new MemoryDiskFloatMLDataSet((long) (memoryStoreSize * crossValidationRate), DTrainUtils.getTestingFile().toString(), this.featureInputsCnt, this.outputNodeCount);
}
// cannot find a good place to close these two data set, using Shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@Override
public void run() {
((MemoryDiskFloatMLDataSet) (AbstractNNWorker.this.trainingData)).close();
((MemoryDiskFloatMLDataSet) (AbstractNNWorker.this.validationData)).close();
}
}));
} catch (IOException e) {
throw new GuaguaRuntimeException(e);
}
}
// create Splitter
String delimiter = context.getProps().getProperty(Constants.SHIFU_OUTPUT_DATA_DELIMITER);
this.splitter = MapReduceUtils.generateShifuOutputSplitter(delimiter);
}
use of ml.shifu.shifu.core.dtrain.gs.GridSearch in project shifu by ShifuML.
the class NNMaster method init.
@SuppressWarnings("unchecked")
@Override
public void init(MasterContext<NNParams, NNParams> context) {
Properties props = context.getProps();
try {
SourceType sourceType = SourceType.valueOf(props.getProperty(CommonConstants.MODELSET_SOURCE_TYPE, SourceType.HDFS.toString()));
this.modelConfig = CommonUtils.loadModelConfig(props.getProperty(CommonConstants.SHIFU_MODEL_CONFIG), sourceType);
this.columnConfigList = CommonUtils.loadColumnConfigList(props.getProperty(CommonConstants.SHIFU_COLUMN_CONFIG), sourceType);
} catch (IOException e) {
throw new RuntimeException(e);
}
int trainerId = Integer.valueOf(context.getProps().getProperty(CommonConstants.SHIFU_TRAINER_ID, "0"));
GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
validParams = this.modelConfig.getTrain().getParams();
if (gs.hasHyperParam()) {
validParams = gs.getParams(trainerId);
LOG.info("Start grid search master with params: {}", validParams);
}
Boolean enabledEarlyStop = DTrainUtils.getBoolean(validParams, CommonConstants.ENABLE_EARLY_STOP, Boolean.FALSE);
if (enabledEarlyStop) {
Double validTolerance = DTrainUtils.getDouble(validParams, CommonConstants.VALIDATION_TOLERANCE, null);
if (validTolerance == null) {
LOG.info("Early Stop is enabled. use WindowEarlyStop");
// windowSize default 20, user should could adjust it
this.earlyStopStrategy = new WindowEarlyStop(context, this.modelConfig, DTrainUtils.getInt(context.getProps(), CommonConstants.SHIFU_TRAIN_EARLYSTOP_WINDOW_SIZE, 20));
} else {
LOG.info("Early Stop is enabled. use ConvergeAndValiToleranceEarlyStop");
Double threshold = this.modelConfig.getTrain().getConvergenceThreshold();
this.earlyStopStrategy = new ConvergeAndValidToleranceEarlyStop(threshold == null ? Double.MIN_VALUE : threshold.doubleValue(), validTolerance);
}
}
Object pObject = validParams.get(CommonConstants.PROPAGATION);
this.propagation = pObject == null ? "Q" : (String) pObject;
this.rawLearningRate = Double.valueOf(validParams.get(CommonConstants.LEARNING_RATE).toString());
Object dropoutRateObj = validParams.get(CommonConstants.DROPOUT_RATE);
if (dropoutRateObj != null) {
this.dropoutRate = Double.valueOf(dropoutRateObj.toString());
}
LOG.info("'dropoutRate' in master is : {}", this.dropoutRate);
Object learningDecayO = validParams.get(CommonConstants.LEARNING_DECAY);
if (learningDecayO != null) {
this.learningDecay = Double.valueOf(learningDecayO.toString());
}
LOG.info("'learningDecay' in master is :{}", learningDecay);
Object momentumO = validParams.get("Momentum");
if (momentumO != null) {
this.momentum = Double.valueOf(momentumO.toString());
}
LOG.info("'momentum' in master is :{}", momentum);
Object adamBeta1O = validParams.get("AdamBeta1");
if (adamBeta1O != null) {
this.adamBeta1 = Double.valueOf(adamBeta1O.toString());
}
LOG.info("'adamBeta1' in master is :{}", adamBeta1);
Object adamBeta2O = validParams.get("AdamBeta2");
if (adamBeta2O != null) {
this.adamBeta2 = Double.valueOf(adamBeta2O.toString());
}
LOG.info("'adamBeta2' in master is :{}", adamBeta2);
this.wgtInit = "default";
Object wgtInitObj = validParams.get(CommonConstants.WEIGHT_INITIALIZER);
if (wgtInitObj != null) {
this.wgtInit = wgtInitObj.toString();
}
this.isContinuousEnabled = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(CommonConstants.CONTINUOUS_TRAINING));
Object rconstant = validParams.get(CommonConstants.REGULARIZED_CONSTANT);
this.regularizedConstant = NumberFormatUtils.getDouble(rconstant == null ? "" : rconstant.toString(), 0d);
// We do not update weight in fixed layers so that we could fine tune other layers of NN
Object fixedLayers2O = validParams.get(CommonConstants.FIXED_LAYERS);
if (fixedLayers2O != null) {
this.fixedLayers = (List<Integer>) fixedLayers2O;
}
LOG.info("Fixed layers in master is :{}", this.fixedLayers.toString());
Object fixedBiasObj = validParams.getOrDefault(CommonConstants.FIXED_BIAS, true);
this.fixedBias = (Boolean) fixedBiasObj;
Object hiddenLayerNumObj = validParams.get(CommonConstants.NUM_HIDDEN_LAYERS);
if (hiddenLayerNumObj != null && StringUtils.isNumeric(hiddenLayerNumObj.toString())) {
this.hiddenLayerNum = Integer.valueOf(hiddenLayerNumObj.toString());
}
LOG.info("hiddenLayerNum in master is :{}", this.hiddenLayerNum);
// check if variables are set final selected
int[] inputOutputIndex = DTrainUtils.getNumericAndCategoricalInputAndOutputCounts(this.columnConfigList);
this.isAfterVarSelect = (inputOutputIndex[3] == 1);
// cache all feature list for sampling features
this.allFeatures = NormalUtils.getAllFeatureList(columnConfigList, isAfterVarSelect);
String subsetStr = context.getProps().getProperty(CommonConstants.SHIFU_NN_FEATURE_SUBSET);
if (StringUtils.isBlank(subsetStr)) {
this.subFeatures = this.allFeatures;
} else {
String[] splits = subsetStr.split(",");
this.subFeatures = new ArrayList<Integer>(splits.length);
for (String split : splits) {
this.subFeatures.add(Integer.parseInt(split));
}
}
// not init but not first iteration, first recover from last master result set from guagua
if (!context.isFirstIteration()) {
NNParams params = context.getMasterResult();
if (params != null && params.getWeights() != null) {
this.globalNNParams.setWeights(params.getWeights());
} else {
// else read from checkpoint
params = initOrRecoverParams(context);
this.globalNNParams.setWeights(params.getWeights());
}
}
}
use of ml.shifu.shifu.core.dtrain.gs.GridSearch in project shifu by ShifuML.
the class LogisticRegressionMaster method init.
@Override
public void init(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) {
loadConfigFiles(context.getProps());
int trainerId = Integer.valueOf(context.getProps().getProperty(CommonConstants.SHIFU_TRAINER_ID, "0"));
GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
validParams = this.modelConfig.getTrain().getParams();
if (gs.hasHyperParam()) {
validParams = gs.getParams(trainerId);
LOG.info("Start grid search master with params: {}", validParams);
}
this.learningRate = Double.valueOf(this.validParams.get(CommonConstants.LEARNING_RATE).toString());
int[] inputOutputIndex = DTrainUtils.getInputOutputCandidateCounts(modelConfig.getNormalizeType(), this.columnConfigList);
this.inputNum = inputOutputIndex[0] == 0 ? inputOutputIndex[2] : inputOutputIndex[0];
Boolean enabledEarlyStop = DTrainUtils.getBoolean(validParams, CommonConstants.ENABLE_EARLY_STOP, Boolean.FALSE);
if (enabledEarlyStop) {
Double validTolerance = DTrainUtils.getDouble(validParams, CommonConstants.VALIDATION_TOLERANCE, null);
if (validTolerance == null) {
LOG.info("Early Stop is enabled. use WindowEarlyStop");
// windowSize default 20, user should could adjust it
this.earlyStopStrategy = new WindowEarlyStop(context, this.modelConfig, DTrainUtils.getInt(context.getProps(), CommonConstants.SHIFU_TRAIN_EARLYSTOP_WINDOW_SIZE, 20));
} else {
LOG.info("Early Stop is enabled. use ConvergeAndValiToleranceEarlyStop");
Double threshold = this.modelConfig.getTrain().getConvergenceThreshold();
this.earlyStopStrategy = new ConvergeAndValidToleranceEarlyStop(threshold == null ? Double.MIN_VALUE : threshold.doubleValue(), validTolerance);
}
}
Object pObject = validParams.get(CommonConstants.PROPAGATION);
this.propagation = pObject == null ? "R" : (String) pObject;
Object rconstant = validParams.get(CommonConstants.REGULARIZED_CONSTANT);
this.regularizedConstant = NumberFormatUtils.getDouble(rconstant == null ? "" : rconstant.toString(), 0d);
this.isContinuousEnabled = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(CommonConstants.CONTINUOUS_TRAINING));
LOG.info("continuousEnabled: {}", this.isContinuousEnabled);
// not initialized and not first iteration, should be fault tolerance, recover state in LogisticRegressionMaster
if (!context.isFirstIteration()) {
LogisticRegressionParams lastMasterResult = context.getMasterResult();
if (lastMasterResult != null && lastMasterResult.getParameters() != null) {
// recover state in current master computable and return to workers
this.weights = lastMasterResult.getParameters();
} else {
// no weights, restarted from the very beginning, this may not happen
this.weights = initWeights().getParameters();
}
}
}
Aggregations