Search in sources :

Example 1 with NormDiscrete

use of org.dmg.pmml.NormDiscreteDocument.NormDiscrete in project knime-core by knime.

the class PMMLOne2ManyTranslator method createDerivedFields.

private DerivedField[] createDerivedFields() {
    List<DerivedField> derivedFields = new ArrayList<DerivedField>();
    for (Map.Entry<String, List<Pair<String, String>>> entry : m_columnMapping.entrySet()) {
        String columnName = entry.getKey();
        String derivedName = m_mapper.getDerivedFieldName(columnName);
        for (Pair<String, String> nameValue : entry.getValue()) {
            DerivedField derivedField = DerivedField.Factory.newInstance();
            derivedField.setName(nameValue.getFirst());
            derivedField.setOptype(OPTYPE.ORDINAL);
            derivedField.setDataType(DATATYPE.INTEGER);
            NormDiscrete normDiscrete = derivedField.addNewNormDiscrete();
            normDiscrete.setField(derivedName);
            normDiscrete.setValue(nameValue.getSecond());
            normDiscrete.setMapMissingTo(0);
            derivedFields.add(derivedField);
        }
    }
    return derivedFields.toArray(new DerivedField[0]);
}
Also used : NormDiscrete(org.dmg.pmml.NormDiscreteDocument.NormDiscrete) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList) DerivedField(org.dmg.pmml.DerivedFieldDocument.DerivedField) Map(java.util.Map)

Example 2 with NormDiscrete

use of org.dmg.pmml.NormDiscreteDocument.NormDiscrete in project knime-core by knime.

the class PMMLOne2ManyTranslator method createDerivedFields.

private DerivedField[] createDerivedFields() {
    List<DerivedField> derivedFields = new ArrayList<DerivedField>();
    for (Map.Entry<String, List<Pair<String, String>>> entry : m_columnMapping.entrySet()) {
        String columnName = entry.getKey();
        String derivedName = m_mapper.getDerivedFieldName(columnName);
        for (Pair<String, String> nameValue : entry.getValue()) {
            DerivedField derivedField = DerivedField.Factory.newInstance();
            derivedField.setName(nameValue.getFirst());
            derivedField.setOptype(OPTYPE.ORDINAL);
            derivedField.setDataType(DATATYPE.INTEGER);
            NormDiscrete normDiscrete = derivedField.addNewNormDiscrete();
            normDiscrete.setField(derivedName);
            normDiscrete.setValue(nameValue.getSecond());
            normDiscrete.setMapMissingTo(0);
            derivedFields.add(derivedField);
        }
    }
    return derivedFields.toArray(new DerivedField[0]);
}
Also used : NormDiscrete(org.dmg.pmml.NormDiscreteDocument.NormDiscrete) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList) DerivedField(org.dmg.pmml.DerivedFieldDocument.DerivedField) Map(java.util.Map)

Example 3 with NormDiscrete

use of org.dmg.pmml.NormDiscreteDocument.NormDiscrete in project knime-core by knime.

the class DataColumnSpecFilterPMMLNodeModel method createPMMLOut.

private PMMLPortObject createPMMLOut(final PMMLPortObject pmmlIn, final DataTableSpec outSpec, final FilterResult res) throws XmlException {
    StringBuffer warningBuffer = new StringBuffer();
    if (pmmlIn == null) {
        return new PMMLPortObject(createPMMLSpec(null, outSpec, res));
    } else {
        PMMLDocument pmmldoc;
        try (LockedSupplier<Document> supplier = pmmlIn.getPMMLValue().getDocumentSupplier()) {
            pmmldoc = PMMLDocument.Factory.parse(supplier.get());
        }
        // Inspect models to check if they use any excluded columns
        List<PMMLModelWrapper> models = PMMLModelWrapper.getModelListFromPMMLDocument(pmmldoc);
        for (PMMLModelWrapper model : models) {
            MiningSchema ms = model.getMiningSchema();
            for (MiningField mf : ms.getMiningFieldList()) {
                if (isExcluded(mf.getName(), res)) {
                    if (warningBuffer.length() != 0) {
                        warningBuffer.append("\n");
                    }
                    warningBuffer.append(model.getModelType().name() + " uses excluded column " + mf.getName());
                }
            }
        }
        ArrayList<String> warningFields = new ArrayList<String>();
        PMML pmml = pmmldoc.getPMML();
        // Now check the transformations if they exist
        if (pmml.getTransformationDictionary() != null) {
            for (DerivedField df : pmml.getTransformationDictionary().getDerivedFieldList()) {
                FieldRef fr = df.getFieldRef();
                if (fr != null && isExcluded(fr.getField(), res)) {
                    warningFields.add(fr.getField());
                }
                Aggregate a = df.getAggregate();
                if (a != null && isExcluded(a.getField(), res)) {
                    warningFields.add(a.getField());
                }
                Apply ap = df.getApply();
                if (ap != null) {
                    for (FieldRef fieldRef : ap.getFieldRefList()) {
                        if (isExcluded(fieldRef.getField(), res)) {
                            warningFields.add(fieldRef.getField());
                            break;
                        }
                    }
                }
                Discretize d = df.getDiscretize();
                if (d != null && isExcluded(d.getField(), res)) {
                    warningFields.add(d.getField());
                }
                MapValues mv = df.getMapValues();
                if (mv != null) {
                    for (FieldColumnPair fcp : mv.getFieldColumnPairList()) {
                        if (isExcluded(fcp.getField(), res)) {
                            warningFields.add(fcp.getField());
                        }
                    }
                }
                NormContinuous nc = df.getNormContinuous();
                if (nc != null && isExcluded(nc.getField(), res)) {
                    warningFields.add(nc.getField());
                }
                NormDiscrete nd = df.getNormDiscrete();
                if (nd != null && isExcluded(nd.getField(), res)) {
                    warningFields.add(nd.getField());
                }
            }
        }
        DataDictionary dict = pmml.getDataDictionary();
        List<DataField> fields = dict.getDataFieldList();
        // Apply filter to spec
        int numFields = 0;
        for (int i = fields.size() - 1; i >= 0; i--) {
            if (isExcluded(fields.get(i).getName(), res)) {
                dict.removeDataField(i);
            } else {
                numFields++;
            }
        }
        dict.setNumberOfFields(new BigInteger(Integer.toString(numFields)));
        pmml.setDataDictionary(dict);
        pmmldoc.setPMML(pmml);
        // generate warnings and set as warning message
        for (String s : warningFields) {
            if (warningBuffer.length() != 0) {
                warningBuffer.append("\n");
            }
            warningBuffer.append("Transformation dictionary uses excluded column " + s);
        }
        if (warningBuffer.length() > 0) {
            setWarningMessage(warningBuffer.toString().trim());
        }
        PMMLPortObject outport = null;
        try {
            outport = new PMMLPortObject(createPMMLSpec(pmmlIn.getSpec(), outSpec, res), pmmldoc);
        } catch (IllegalArgumentException e) {
            if (res.getIncludes().length == 0) {
                throw new IllegalArgumentException("Excluding all columns produces invalid PMML", e);
            } else {
                throw e;
            }
        }
        return outport;
    }
}
Also used : MiningField(org.dmg.pmml.MiningFieldDocument.MiningField) NormContinuous(org.dmg.pmml.NormContinuousDocument.NormContinuous) Apply(org.dmg.pmml.ApplyDocument.Apply) ArrayList(java.util.ArrayList) FieldColumnPair(org.dmg.pmml.FieldColumnPairDocument.FieldColumnPair) PMMLDocument(org.dmg.pmml.PMMLDocument) Document(org.w3c.dom.Document) MapValues(org.dmg.pmml.MapValuesDocument.MapValues) Discretize(org.dmg.pmml.DiscretizeDocument.Discretize) FieldRef(org.dmg.pmml.FieldRefDocument.FieldRef) DataDictionary(org.dmg.pmml.DataDictionaryDocument.DataDictionary) PMMLModelWrapper(org.knime.core.node.port.pmml.PMMLModelWrapper) NormDiscrete(org.dmg.pmml.NormDiscreteDocument.NormDiscrete) MiningSchema(org.dmg.pmml.MiningSchemaDocument.MiningSchema) DataField(org.dmg.pmml.DataFieldDocument.DataField) PMMLPortObject(org.knime.core.node.port.pmml.PMMLPortObject) PMML(org.dmg.pmml.PMMLDocument.PMML) BigInteger(java.math.BigInteger) PMMLDocument(org.dmg.pmml.PMMLDocument) Aggregate(org.dmg.pmml.AggregateDocument.Aggregate) DerivedField(org.dmg.pmml.DerivedFieldDocument.DerivedField)

Example 4 with NormDiscrete

use of org.dmg.pmml.NormDiscreteDocument.NormDiscrete in project knime-core by knime.

the class PMMLNeuralNetworkTranslator method addOutputLayer.

/**
 * Writes the PMML output layer of the MLP.
 *
 * @param nnModel
 *            the neural network model.
 * @param mlp
 *            the underlying {@link MultiLayerPerceptron}.
 * @param spec
 *            the port object spec
 */
protected void addOutputLayer(final NeuralNetwork nnModel, final MultiLayerPerceptron mlp, final PMMLPortObjectSpec spec) {
    int lastlayer = mlp.getNrLayers() - 1;
    String targetCol = spec.getTargetFields().iterator().next();
    Layer outputlayer = mlp.getLayer(lastlayer);
    Perceptron[] outputperceptrons = outputlayer.getPerceptrons();
    HashMap<DataCell, Integer> outputmap = mlp.getClassMapping();
    NeuralOutputs neuralOuts = nnModel.addNewNeuralOutputs();
    neuralOuts.setNumberOfOutputs(BigInteger.valueOf(outputperceptrons.length));
    for (int i = 0; i < outputperceptrons.length; i++) {
        NeuralOutput neuralOutput = neuralOuts.addNewNeuralOutput();
        neuralOutput.setOutputNeuron(lastlayer + "," + i);
        // search corresponding output value
        String colname = "";
        for (Entry<DataCell, Integer> e : outputmap.entrySet()) {
            if (e.getValue().equals(i)) {
                colname = ((StringValue) e.getKey()).getStringValue();
            }
        }
        DerivedField df = neuralOutput.addNewDerivedField();
        df.setOptype(OPTYPE.CATEGORICAL);
        df.setDataType(DATATYPE.STRING);
        if (mlp.getMode() == MultiLayerPerceptron.CLASSIFICATION_MODE) {
            df.setOptype(OPTYPE.CATEGORICAL);
            df.setDataType(DATATYPE.STRING);
        } else if (mlp.getMode() == MultiLayerPerceptron.REGRESSION_MODE) {
            df.setOptype(OPTYPE.CONTINUOUS);
            df.setDataType(DATATYPE.DOUBLE);
        }
        if (mlp.getMode() == MultiLayerPerceptron.CLASSIFICATION_MODE) {
            NormDiscrete normDiscrete = df.addNewNormDiscrete();
            normDiscrete.setField(targetCol);
            normDiscrete.setValue(colname);
        } else if (mlp.getMode() == MultiLayerPerceptron.REGRESSION_MODE) {
            FieldRef fieldRef = df.addNewFieldRef();
            fieldRef.setField(targetCol);
        }
    }
}
Also used : NeuralOutputs(org.dmg.pmml.NeuralOutputsDocument.NeuralOutputs) FieldRef(org.dmg.pmml.FieldRefDocument.FieldRef) NeuralLayer(org.dmg.pmml.NeuralLayerDocument.NeuralLayer) Layer(org.knime.base.data.neural.Layer) InputLayer(org.knime.base.data.neural.InputLayer) HiddenLayer(org.knime.base.data.neural.HiddenLayer) NeuralOutput(org.dmg.pmml.NeuralOutputDocument.NeuralOutput) BigInteger(java.math.BigInteger) NormDiscrete(org.dmg.pmml.NormDiscreteDocument.NormDiscrete) SigmoidPerceptron(org.knime.base.data.neural.SigmoidPerceptron) MultiLayerPerceptron(org.knime.base.data.neural.MultiLayerPerceptron) Perceptron(org.knime.base.data.neural.Perceptron) InputPerceptron(org.knime.base.data.neural.InputPerceptron) DataCell(org.knime.core.data.DataCell) DerivedField(org.dmg.pmml.DerivedFieldDocument.DerivedField)

Example 5 with NormDiscrete

use of org.dmg.pmml.NormDiscreteDocument.NormDiscrete in project knime-core by knime.

the class PMMLNeuralNetworkTranslator method addOutputLayer.

/**
 * Writes the PMML output layer of the MLP.
 *
 * @param nnModel
 *            the neural network model.
 * @param mlp
 *            the underlying {@link MultiLayerPerceptron}.
 * @param spec
 *            the port object spec
 */
protected void addOutputLayer(final NeuralNetwork nnModel, final MultiLayerPerceptron mlp, final PMMLPortObjectSpec spec) {
    int lastlayer = mlp.getNrLayers() - 1;
    String targetCol = spec.getTargetFields().iterator().next();
    Layer outputlayer = mlp.getLayer(lastlayer);
    Perceptron[] outputperceptrons = outputlayer.getPerceptrons();
    HashMap<DataCell, Integer> outputmap = mlp.getClassMapping();
    NeuralOutputs neuralOuts = nnModel.addNewNeuralOutputs();
    neuralOuts.setNumberOfOutputs(BigInteger.valueOf(outputperceptrons.length));
    for (int i = 0; i < outputperceptrons.length; i++) {
        NeuralOutput neuralOutput = neuralOuts.addNewNeuralOutput();
        neuralOutput.setOutputNeuron(lastlayer + "," + i);
        // search corresponding output value
        String colname = "";
        for (Entry<DataCell, Integer> e : outputmap.entrySet()) {
            if (e.getValue().equals(i)) {
                colname = ((StringValue) e.getKey()).getStringValue();
            }
        }
        DerivedField df = neuralOutput.addNewDerivedField();
        df.setOptype(OPTYPE.CATEGORICAL);
        df.setDataType(DATATYPE.STRING);
        if (mlp.getMode() == MultiLayerPerceptron.CLASSIFICATION_MODE) {
            df.setOptype(OPTYPE.CATEGORICAL);
            df.setDataType(DATATYPE.STRING);
        } else if (mlp.getMode() == MultiLayerPerceptron.REGRESSION_MODE) {
            df.setOptype(OPTYPE.CONTINUOUS);
            df.setDataType(DATATYPE.DOUBLE);
        }
        if (mlp.getMode() == MultiLayerPerceptron.CLASSIFICATION_MODE) {
            NormDiscrete normDiscrete = df.addNewNormDiscrete();
            normDiscrete.setField(targetCol);
            normDiscrete.setValue(colname);
        } else if (mlp.getMode() == MultiLayerPerceptron.REGRESSION_MODE) {
            FieldRef fieldRef = df.addNewFieldRef();
            fieldRef.setField(targetCol);
        }
    }
}
Also used : NeuralOutputs(org.dmg.pmml.NeuralOutputsDocument.NeuralOutputs) FieldRef(org.dmg.pmml.FieldRefDocument.FieldRef) NeuralLayer(org.dmg.pmml.NeuralLayerDocument.NeuralLayer) Layer(org.knime.base.data.neural.Layer) InputLayer(org.knime.base.data.neural.InputLayer) HiddenLayer(org.knime.base.data.neural.HiddenLayer) NeuralOutput(org.dmg.pmml.NeuralOutputDocument.NeuralOutput) BigInteger(java.math.BigInteger) NormDiscrete(org.dmg.pmml.NormDiscreteDocument.NormDiscrete) SigmoidPerceptron(org.knime.base.data.neural.SigmoidPerceptron) MultiLayerPerceptron(org.knime.base.data.neural.MultiLayerPerceptron) Perceptron(org.knime.base.data.neural.Perceptron) InputPerceptron(org.knime.base.data.neural.InputPerceptron) DataCell(org.knime.core.data.DataCell) DerivedField(org.dmg.pmml.DerivedFieldDocument.DerivedField)

Aggregations

DerivedField (org.dmg.pmml.DerivedFieldDocument.DerivedField)5 NormDiscrete (org.dmg.pmml.NormDiscreteDocument.NormDiscrete)5 BigInteger (java.math.BigInteger)3 ArrayList (java.util.ArrayList)3 FieldRef (org.dmg.pmml.FieldRefDocument.FieldRef)3 List (java.util.List)2 Map (java.util.Map)2 NeuralLayer (org.dmg.pmml.NeuralLayerDocument.NeuralLayer)2 NeuralOutput (org.dmg.pmml.NeuralOutputDocument.NeuralOutput)2 NeuralOutputs (org.dmg.pmml.NeuralOutputsDocument.NeuralOutputs)2 HiddenLayer (org.knime.base.data.neural.HiddenLayer)2 InputLayer (org.knime.base.data.neural.InputLayer)2 InputPerceptron (org.knime.base.data.neural.InputPerceptron)2 Layer (org.knime.base.data.neural.Layer)2 MultiLayerPerceptron (org.knime.base.data.neural.MultiLayerPerceptron)2 Perceptron (org.knime.base.data.neural.Perceptron)2 SigmoidPerceptron (org.knime.base.data.neural.SigmoidPerceptron)2 DataCell (org.knime.core.data.DataCell)2 Aggregate (org.dmg.pmml.AggregateDocument.Aggregate)1 Apply (org.dmg.pmml.ApplyDocument.Apply)1