Examples with NormContinuous org.dmg.pmml.NormContinuous used on opensource projects

Example 6 with NormContinuous

use of org.dmg.pmml.NormContinuous in project shifu by ShifuML.

the class PMMLAdapterCommonUtil method getRegressionTable.

/**
 * Generate Regression Table based on the weight list, intercept and partial
 * PMML model
 *
 * @param weights
 *            weight list for the Regression Table
 * @param intercept
 *            the intercept
 * @param pmmlModel
 *            partial PMMl model
 * @return regression model instance
 */
public static RegressionModel getRegressionTable(final double[] weights, final double intercept, RegressionModel pmmlModel) {
    RegressionTable table = new RegressionTable();
    MiningSchema schema = pmmlModel.getMiningSchema();
    // TODO may not need target field in LRModel
    pmmlModel.setMiningFunction(MiningFunction.REGRESSION);
    pmmlModel.setNormalizationMethod(NormalizationMethod.LOGIT);
    List<String> outputFields = getSchemaFieldViaUsageType(schema, UsageType.TARGET);
    // TODO only one outputField, what if we have more than one outputField
    pmmlModel.setTargetFieldName(new FieldName(outputFields.get(0)));
    table.setTargetCategory(outputFields.get(0));
    List<String> activeFields = getSchemaFieldViaUsageType(schema, UsageType.ACTIVE);
    int index = 0;
    for (DerivedField dField : pmmlModel.getLocalTransformations().getDerivedFields()) {
        Expression expression = dField.getExpression();
        if (expression instanceof NormContinuous) {
            NormContinuous norm = (NormContinuous) expression;
            if (activeFields.contains(norm.getField().getValue()))
                table.addNumericPredictors(new NumericPredictor(dField.getName(), weights[index++]));
        }
    }
    pmmlModel.addRegressionTables(table);
    return pmmlModel;
}

Example 7 with NormContinuous

use of org.dmg.pmml.NormContinuous in project shifu by ShifuML.

the class PMMLLRModelBuilder method adaptMLModelToPMML.

public RegressionModel adaptMLModelToPMML(ml.shifu.shifu.core.LR lr, RegressionModel pmmlModel) {
    pmmlModel.setNormalizationMethod(NormalizationMethod.LOGIT);
    pmmlModel.setMiningFunction(MiningFunction.REGRESSION);
    RegressionTable table = new RegressionTable();
    table.setIntercept(lr.getBias());
    LocalTransformations lt = pmmlModel.getLocalTransformations();
    List<DerivedField> df = lt.getDerivedFields();
    HashMap<FieldName, FieldName> miningTransformMap = new HashMap<FieldName, FieldName>();
    for (DerivedField dField : df) {
        // Apply z-scale normalization on numerical variables
        if (dField.getExpression() instanceof NormContinuous) {
            miningTransformMap.put(((NormContinuous) dField.getExpression()).getField(), dField.getName());
        } else // Apply bin map on categorical variables
        if (dField.getExpression() instanceof MapValues) {
            miningTransformMap.put(((MapValues) dField.getExpression()).getFieldColumnPairs().get(0).getField(), dField.getName());
        } else if (dField.getExpression() instanceof Discretize) {
            miningTransformMap.put(((Discretize) dField.getExpression()).getField(), dField.getName());
        }
    }
    List<MiningField> miningList = pmmlModel.getMiningSchema().getMiningFields();
    int index = 0;
    for (int i = 0; i < miningList.size(); i++) {
        MiningField mField = miningList.get(i);
        if (mField.getUsageType() != UsageType.ACTIVE)
            continue;
        FieldName mFieldName = mField.getName();
        FieldName fName = mFieldName;
        while (miningTransformMap.containsKey(fName)) {
            fName = miningTransformMap.get(fName);
        }
        NumericPredictor np = new NumericPredictor();
        np.setName(fName);
        np.setCoefficient(lr.getWeights()[index++]);
        table.addNumericPredictors(np);
    }
    pmmlModel.addRegressionTables(table);
    return pmmlModel;
}

Example 8 with NormContinuous

use of org.dmg.pmml.NormContinuous in project shifu by ShifuML.

the class WoeZscoreLocalTransformCreator method createCategoricalDerivedField.

/**
 * Create @DerivedField for categorical variable
 *
 * @param config - ColumnConfig for categorical variable
 * @param cutoff - cutoff for normalization
 * @return DerivedField for variable
 */
@Override
protected List<DerivedField> createCategoricalDerivedField(ColumnConfig config, double cutoff, ModelNormalizeConf.NormType normType) {
    List<DerivedField> derivedFields = new ArrayList<DerivedField>();
    DerivedField derivedField = super.createCategoricalDerivedField(config, cutoff, ModelNormalizeConf.NormType.WOE).get(0);
    derivedFields.add(derivedField);
    double[] meanAndStdDev = Normalizer.calculateWoeMeanAndStdDev(config, isWeightedNorm);
    // added capping logic to linearNorm
    LinearNorm from = new LinearNorm().setOrig(meanAndStdDev[0] - meanAndStdDev[1] * cutoff).setNorm(-cutoff);
    LinearNorm to = new LinearNorm().setOrig(meanAndStdDev[0] + meanAndStdDev[1] * cutoff).setNorm(cutoff);
    NormContinuous normContinuous = new NormContinuous();
    normContinuous.setField(FieldName.create(derivedField.getName().getValue()));
    normContinuous.addLinearNorms(from, to);
    normContinuous.setMapMissingTo(0.0);
    normContinuous.setOutliers(OutlierTreatmentMethod.AS_EXTREME_VALUES);
    // derived field name is consisted of FieldName and "_zscl"
    derivedFields.add(new DerivedField(OpType.CONTINUOUS, DataType.DOUBLE).setName(FieldName.create(genPmmlColumnName(NormalUtils.getSimpleColumnName(config.getColumnName()), normType))).setExpression(normContinuous));
    return derivedFields;
}

Example 9 with NormContinuous

use of org.dmg.pmml.NormContinuous in project shifu by ShifuML.

the class NeuralNetworkModelIntegrator method getNeuralInputs.

private NeuralInputs getNeuralInputs(final NeuralNetwork model) {
    NeuralInputs nnInputs = new NeuralInputs();
    // get HashMap for local transform and MiningSchema fields
    HashMap<FieldName, FieldName> reversMiningTransformMap = new HashMap<FieldName, FieldName>();
    HashMap<FieldName, List<FieldName>> treeMapOfTransform = new HashMap<FieldName, List<FieldName>>();
    for (DerivedField dField : model.getLocalTransformations().getDerivedFields()) {
        // Apply z-scale normalization on numerical variables
        FieldName parentField = null;
        if (dField.getExpression() instanceof NormContinuous) {
            parentField = ((NormContinuous) dField.getExpression()).getField();
            reversMiningTransformMap.put(dField.getName(), parentField);
        } else // Apply bin map on categorical variables
        if (dField.getExpression() instanceof MapValues) {
            parentField = ((MapValues) dField.getExpression()).getFieldColumnPairs().get(0).getField();
            reversMiningTransformMap.put(dField.getName(), parentField);
        } else if (dField.getExpression() instanceof Discretize) {
            parentField = ((Discretize) dField.getExpression()).getField();
            reversMiningTransformMap.put(dField.getName(), parentField);
        }
        List<FieldName> fieldNames = treeMapOfTransform.get(parentField);
        if (fieldNames == null) {
            fieldNames = new ArrayList<FieldName>();
        }
        fieldNames.add(dField.getName());
        treeMapOfTransform.put(parentField, fieldNames);
    }
    // comment here
    List<MiningField> miningList = model.getMiningSchema().getMiningFields();
    int index = 0;
    for (DerivedField dField : model.getLocalTransformations().getDerivedFields()) {
        List<FieldName> list = treeMapOfTransform.get(dField.getName());
        boolean isLeaf = (list == null || list.size() == 0);
        FieldName root = getRoot(dField.getName(), reversMiningTransformMap);
        if (isLeaf && isRootInMiningList(root, miningList)) {
            DerivedField field = new DerivedField(OpType.CONTINUOUS, DataType.DOUBLE).setName(dField.getName()).setExpression(new FieldRef(dField.getName()));
            nnInputs.addNeuralInputs(new NeuralInput("0," + (index++), field));
        }
    }
    DerivedField field = new DerivedField(OpType.CONTINUOUS, DataType.DOUBLE).setName(new FieldName(PluginConstants.biasValue)).setExpression(new FieldRef(new FieldName(PluginConstants.biasValue)));
    nnInputs.addNeuralInputs(new NeuralInput(PluginConstants.biasValue, field));
    return nnInputs;
}