Examples with ConvergeAndValidToleranceEarlyStop ml.shifu.shifu.core.dtrain.earlystop.ConvergeAndValidToleranceEarlyStop used on opensource projects

Example 1 with ConvergeAndValidToleranceEarlyStop

use of ml.shifu.shifu.core.dtrain.earlystop.ConvergeAndValidToleranceEarlyStop in project shifu by ShifuML.

the class NNMaster method init.

@SuppressWarnings("unchecked")
@Override
public void init(MasterContext<NNParams, NNParams> context) {
    Properties props = context.getProps();
    try {
        SourceType sourceType = SourceType.valueOf(props.getProperty(CommonConstants.MODELSET_SOURCE_TYPE, SourceType.HDFS.toString()));
        this.modelConfig = CommonUtils.loadModelConfig(props.getProperty(CommonConstants.SHIFU_MODEL_CONFIG), sourceType);
        this.columnConfigList = CommonUtils.loadColumnConfigList(props.getProperty(CommonConstants.SHIFU_COLUMN_CONFIG), sourceType);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    int trainerId = Integer.valueOf(context.getProps().getProperty(CommonConstants.SHIFU_TRAINER_ID, "0"));
    GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
    validParams = this.modelConfig.getTrain().getParams();
    if (gs.hasHyperParam()) {
        validParams = gs.getParams(trainerId);
        LOG.info("Start grid search master with params: {}", validParams);
    }
    Boolean enabledEarlyStop = DTrainUtils.getBoolean(validParams, CommonConstants.ENABLE_EARLY_STOP, Boolean.FALSE);
    if (enabledEarlyStop) {
        Double validTolerance = DTrainUtils.getDouble(validParams, CommonConstants.VALIDATION_TOLERANCE, null);
        if (validTolerance == null) {
            LOG.info("Early Stop is enabled. use WindowEarlyStop");
            // windowSize default 20, user should could adjust it
            this.earlyStopStrategy = new WindowEarlyStop(context, this.modelConfig, DTrainUtils.getInt(context.getProps(), CommonConstants.SHIFU_TRAIN_EARLYSTOP_WINDOW_SIZE, 20));
        } else {
            LOG.info("Early Stop is enabled. use ConvergeAndValiToleranceEarlyStop");
            Double threshold = this.modelConfig.getTrain().getConvergenceThreshold();
            this.earlyStopStrategy = new ConvergeAndValidToleranceEarlyStop(threshold == null ? Double.MIN_VALUE : threshold.doubleValue(), validTolerance);
        }
    }
    Object pObject = validParams.get(CommonConstants.PROPAGATION);
    this.propagation = pObject == null ? "Q" : (String) pObject;
    this.rawLearningRate = Double.valueOf(validParams.get(CommonConstants.LEARNING_RATE).toString());
    Object dropoutRateObj = validParams.get(CommonConstants.DROPOUT_RATE);
    if (dropoutRateObj != null) {
        this.dropoutRate = Double.valueOf(dropoutRateObj.toString());
    }
    LOG.info("'dropoutRate' in master is : {}", this.dropoutRate);
    Object learningDecayO = validParams.get(CommonConstants.LEARNING_DECAY);
    if (learningDecayO != null) {
        this.learningDecay = Double.valueOf(learningDecayO.toString());
    }
    LOG.info("'learningDecay' in master is :{}", learningDecay);
    Object momentumO = validParams.get("Momentum");
    if (momentumO != null) {
        this.momentum = Double.valueOf(momentumO.toString());
    }
    LOG.info("'momentum' in master is :{}", momentum);
    Object adamBeta1O = validParams.get("AdamBeta1");
    if (adamBeta1O != null) {
        this.adamBeta1 = Double.valueOf(adamBeta1O.toString());
    }
    LOG.info("'adamBeta1' in master is :{}", adamBeta1);
    Object adamBeta2O = validParams.get("AdamBeta2");
    if (adamBeta2O != null) {
        this.adamBeta2 = Double.valueOf(adamBeta2O.toString());
    }
    LOG.info("'adamBeta2' in master is :{}", adamBeta2);
    this.wgtInit = "default";
    Object wgtInitObj = validParams.get(CommonConstants.WEIGHT_INITIALIZER);
    if (wgtInitObj != null) {
        this.wgtInit = wgtInitObj.toString();
    }
    this.isContinuousEnabled = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(CommonConstants.CONTINUOUS_TRAINING));
    Object rconstant = validParams.get(CommonConstants.REGULARIZED_CONSTANT);
    this.regularizedConstant = NumberFormatUtils.getDouble(rconstant == null ? "" : rconstant.toString(), 0d);
    // We do not update weight in fixed layers so that we could fine tune other layers of NN
    Object fixedLayers2O = validParams.get(CommonConstants.FIXED_LAYERS);
    if (fixedLayers2O != null) {
        this.fixedLayers = (List<Integer>) fixedLayers2O;
    }
    LOG.info("Fixed layers in master is :{}", this.fixedLayers.toString());
    Object fixedBiasObj = validParams.getOrDefault(CommonConstants.FIXED_BIAS, true);
    this.fixedBias = (Boolean) fixedBiasObj;
    Object hiddenLayerNumObj = validParams.get(CommonConstants.NUM_HIDDEN_LAYERS);
    if (hiddenLayerNumObj != null && StringUtils.isNumeric(hiddenLayerNumObj.toString())) {
        this.hiddenLayerNum = Integer.valueOf(hiddenLayerNumObj.toString());
    }
    LOG.info("hiddenLayerNum in master is :{}", this.hiddenLayerNum);
    // check if variables are set final selected
    int[] inputOutputIndex = DTrainUtils.getNumericAndCategoricalInputAndOutputCounts(this.columnConfigList);
    this.isAfterVarSelect = (inputOutputIndex[3] == 1);
    // cache all feature list for sampling features
    this.allFeatures = NormalUtils.getAllFeatureList(columnConfigList, isAfterVarSelect);
    String subsetStr = context.getProps().getProperty(CommonConstants.SHIFU_NN_FEATURE_SUBSET);
    if (StringUtils.isBlank(subsetStr)) {
        this.subFeatures = this.allFeatures;
    } else {
        String[] splits = subsetStr.split(",");
        this.subFeatures = new ArrayList<Integer>(splits.length);
        for (String split : splits) {
            this.subFeatures.add(Integer.parseInt(split));
        }
    }
    // not init but not first iteration, first recover from last master result set from guagua
    if (!context.isFirstIteration()) {
        NNParams params = context.getMasterResult();
        if (params != null && params.getWeights() != null) {
            this.globalNNParams.setWeights(params.getWeights());
        } else {
            // else read from checkpoint
            params = initOrRecoverParams(context);
            this.globalNNParams.setWeights(params.getWeights());
        }
    }
}

Example 2 with ConvergeAndValidToleranceEarlyStop

use of ml.shifu.shifu.core.dtrain.earlystop.ConvergeAndValidToleranceEarlyStop in project shifu by ShifuML.

the class LogisticRegressionMaster method init.

@Override
public void init(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) {
    loadConfigFiles(context.getProps());
    int trainerId = Integer.valueOf(context.getProps().getProperty(CommonConstants.SHIFU_TRAINER_ID, "0"));
    GridSearch gs = new GridSearch(modelConfig.getTrain().getParams(), modelConfig.getTrain().getGridConfigFileContent());
    validParams = this.modelConfig.getTrain().getParams();
    if (gs.hasHyperParam()) {
        validParams = gs.getParams(trainerId);
        LOG.info("Start grid search master with params: {}", validParams);
    }
    this.learningRate = Double.valueOf(this.validParams.get(CommonConstants.LEARNING_RATE).toString());
    int[] inputOutputIndex = DTrainUtils.getInputOutputCandidateCounts(modelConfig.getNormalizeType(), this.columnConfigList);
    this.inputNum = inputOutputIndex[0] == 0 ? inputOutputIndex[2] : inputOutputIndex[0];
    Boolean enabledEarlyStop = DTrainUtils.getBoolean(validParams, CommonConstants.ENABLE_EARLY_STOP, Boolean.FALSE);
    if (enabledEarlyStop) {
        Double validTolerance = DTrainUtils.getDouble(validParams, CommonConstants.VALIDATION_TOLERANCE, null);
        if (validTolerance == null) {
            LOG.info("Early Stop is enabled. use WindowEarlyStop");
            // windowSize default 20, user should could adjust it
            this.earlyStopStrategy = new WindowEarlyStop(context, this.modelConfig, DTrainUtils.getInt(context.getProps(), CommonConstants.SHIFU_TRAIN_EARLYSTOP_WINDOW_SIZE, 20));
        } else {
            LOG.info("Early Stop is enabled. use ConvergeAndValiToleranceEarlyStop");
            Double threshold = this.modelConfig.getTrain().getConvergenceThreshold();
            this.earlyStopStrategy = new ConvergeAndValidToleranceEarlyStop(threshold == null ? Double.MIN_VALUE : threshold.doubleValue(), validTolerance);
        }
    }
    Object pObject = validParams.get(CommonConstants.PROPAGATION);
    this.propagation = pObject == null ? "R" : (String) pObject;
    Object rconstant = validParams.get(CommonConstants.REGULARIZED_CONSTANT);
    this.regularizedConstant = NumberFormatUtils.getDouble(rconstant == null ? "" : rconstant.toString(), 0d);
    this.isContinuousEnabled = Boolean.TRUE.toString().equalsIgnoreCase(context.getProps().getProperty(CommonConstants.CONTINUOUS_TRAINING));
    LOG.info("continuousEnabled: {}", this.isContinuousEnabled);
    // not initialized and not first iteration, should be fault tolerance, recover state in LogisticRegressionMaster
    if (!context.isFirstIteration()) {
        LogisticRegressionParams lastMasterResult = context.getMasterResult();
        if (lastMasterResult != null && lastMasterResult.getParameters() != null) {
            // recover state in current master computable and return to workers
            this.weights = lastMasterResult.getParameters();
        } else {
            // no weights, restarted from the very beginning, this may not happen
            this.weights = initWeights().getParameters();
        }
    }
}