use of org.dmg.pmml.MapValuesDocument.MapValues in project knime-core by knime.
the class CategoryToNumberApplyNodeModel method createRearranger.
/**
* Creates a rearranger that processes the derived fields with MapValues
* in the given model.
*/
private ColumnRearranger createRearranger(final DataTableSpec spec, final PMMLPortObject model) {
// Retrieve columns with string data in the spec
Set<String> stringCols = new LinkedHashSet<String>();
Set<String> otherCols = new LinkedHashSet<String>();
for (DataColumnSpec colSpec : spec) {
if (colSpec.getType().isCompatible(StringValue.class)) {
stringCols.add(colSpec.getName());
} else {
otherCols.add(colSpec.getName());
}
}
if (stringCols.isEmpty()) {
if (null == model) {
// during configure
setWarningMessage("No columns to process.");
} else {
// during execute
setWarningMessage("No columns to process, returning input.");
}
}
// The map values in the model if present
Map<String, DerivedField> mapValues = null != model ? getMapValues(model) : Collections.<String, DerivedField>emptyMap();
// Create rearranger
ColumnRearranger rearranger = new ColumnRearranger(spec);
for (String col : mapValues.keySet()) {
DerivedField derivedField = mapValues.get(col);
MapValues map = derivedField.getMapValues();
// this PMML MapValues model is found but has wrong type.
if (null != model && otherCols.contains(col)) {
String outColumn = null == derivedField.getDisplayName() || derivedField.getDisplayName().trim().isEmpty() ? derivedField.getName() : derivedField.getDisplayName();
LOGGER.warn("Cannot create column \"" + outColumn + "\" since the input column \"" + col + "\" is not of type StringValue.");
continue;
}
// this PMML MapValues model.
if (null != model && !stringCols.contains(col)) {
String outColumn = null == derivedField.getDisplayName() || derivedField.getDisplayName().trim().isEmpty() ? derivedField.getName() : derivedField.getDisplayName();
LOGGER.warn("Cannot create column \"" + outColumn + "\" since the column \"" + col + "\" is not in the input.");
continue;
}
CategoryToNumberApplyCellFactory factory = new CategoryToNumberApplyCellFactory(spec, col, m_settings, map);
if (m_settings.getAppendColumns()) {
rearranger.append(factory);
} else {
rearranger.replace(factory, col);
}
}
return rearranger;
}
use of org.dmg.pmml.MapValuesDocument.MapValues in project knime-core by knime.
the class PMMLMapValuesTranslator method createDerivedFields.
private DerivedField[] createDerivedFields() {
DerivedField df = DerivedField.Factory.newInstance();
df.setExtensionArray(createSummaryExtension());
/* The field name must be retrieved before creating a new derived
* name for this derived field as the map only contains the
* current mapping. */
String fieldName = m_mapper.getDerivedFieldName(m_config.getInColumn());
if (m_config.getInColumn().equals(m_config.getOutColumn())) {
String name = m_config.getInColumn();
df.setDisplayName(name);
df.setName(m_mapper.createDerivedFieldName(name));
} else {
df.setName(m_config.getOutColumn());
}
df.setOptype(m_config.getOpType());
df.setDataType(m_config.getOutDataType());
MapValues mapValues = df.addNewMapValues();
// the element in the InlineTable representing the output column
// Use dummy name instead of m_config.getOutColumn() since the
// input column could contain characters that are not allowed in XML
final QName xmlOut = new QName("http://www.dmg.org/PMML-4_0", "out");
mapValues.setOutputColumn(xmlOut.getLocalPart());
mapValues.setDataType(m_config.getOutDataType());
if (!m_config.getDefaultValue().isMissing()) {
mapValues.setDefaultValue(m_config.getDefaultValue().toString());
}
if (!m_config.getMapMissingTo().isMissing()) {
mapValues.setMapMissingTo(m_config.getMapMissingTo().toString());
}
// the mapping of input field <-> element in the InlineTable
FieldColumnPair fieldColPair = mapValues.addNewFieldColumnPair();
fieldColPair.setField(fieldName);
// Use dummy name instead of m_config.getInColumn() since the
// input column could contain characters that are not allowed in XML
final QName xmlIn = new QName("http://www.dmg.org/PMML-4_0", "in");
fieldColPair.setColumn(xmlIn.getLocalPart());
InlineTable table = mapValues.addNewInlineTable();
for (Entry<DataCell, ? extends DataCell> entry : m_config.getEntries().entrySet()) {
Row row = table.addNewRow();
XmlCursor cursor = row.newCursor();
cursor.toNextToken();
cursor.insertElementWithText(xmlIn, entry.getKey().toString());
cursor.insertElementWithText(xmlOut, entry.getValue().toString());
cursor.dispose();
}
return new DerivedField[] { df };
}
use of org.dmg.pmml.MapValuesDocument.MapValues in project knime-core by knime.
the class NumberToCategoryApplyNodeModel method getMapValues.
/**
* @param model the PMML model
* @return the field in the first FieldColumnPair of the MapValues mapped
* to the MapValues Model
*/
private Map<String, DerivedField> getMapValues(final PMMLPortObject model) {
Map<String, DerivedField> mapValues = new LinkedHashMap<String, DerivedField>();
DerivedField[] derivedFields = model.getDerivedFields();
for (DerivedField derivedField : derivedFields) {
MapValues map = derivedField.getMapValues();
if (null != map) {
// This is the field name the mapValues is based on
String name = derivedField.getDisplayName();
if (name == null) {
name = derivedField.getName();
}
mapValues.put(name, derivedField);
}
}
return mapValues;
}
use of org.dmg.pmml.MapValuesDocument.MapValues in project knime-core by knime.
the class PMMLMapValuesTranslator method createDerivedFields.
private DerivedField[] createDerivedFields() {
DerivedField df = DerivedField.Factory.newInstance();
df.setExtensionArray(createSummaryExtension());
/* The field name must be retrieved before creating a new derived
* name for this derived field as the map only contains the
* current mapping. */
String fieldName = m_mapper.getDerivedFieldName(m_config.getInColumn());
if (m_config.getInColumn().equals(m_config.getOutColumn())) {
String name = m_config.getInColumn();
df.setDisplayName(name);
df.setName(m_mapper.createDerivedFieldName(name));
} else {
df.setName(m_config.getOutColumn());
}
df.setOptype(m_config.getOpType());
df.setDataType(m_config.getOutDataType());
MapValues mapValues = df.addNewMapValues();
// the element in the InlineTable representing the output column
// Use dummy name instead of m_config.getOutColumn() since the
// input column could contain characters that are not allowed in XML
final QName xmlOut = new QName("http://www.dmg.org/PMML-4_0", "out");
mapValues.setOutputColumn(xmlOut.getLocalPart());
mapValues.setDataType(m_config.getOutDataType());
if (!m_config.getDefaultValue().isMissing()) {
mapValues.setDefaultValue(m_config.getDefaultValue().toString());
}
if (!m_config.getMapMissingTo().isMissing()) {
mapValues.setMapMissingTo(m_config.getMapMissingTo().toString());
}
// the mapping of input field <-> element in the InlineTable
FieldColumnPair fieldColPair = mapValues.addNewFieldColumnPair();
fieldColPair.setField(fieldName);
// Use dummy name instead of m_config.getInColumn() since the
// input column could contain characters that are not allowed in XML
final QName xmlIn = new QName("http://www.dmg.org/PMML-4_0", "in");
fieldColPair.setColumn(xmlIn.getLocalPart());
InlineTable table = mapValues.addNewInlineTable();
for (Entry<DataCell, ? extends DataCell> entry : m_config.getEntries().entrySet()) {
Row row = table.addNewRow();
XmlCursor cursor = row.newCursor();
cursor.toNextToken();
cursor.insertElementWithText(xmlIn, entry.getKey().toString());
cursor.insertElementWithText(xmlOut, entry.getValue().toString());
cursor.dispose();
}
return new DerivedField[] { df };
}
use of org.dmg.pmml.MapValuesDocument.MapValues in project knime-core by knime.
the class DataColumnSpecFilterPMMLNodeModel method createPMMLOut.
private PMMLPortObject createPMMLOut(final PMMLPortObject pmmlIn, final DataTableSpec outSpec, final FilterResult res) throws XmlException {
StringBuffer warningBuffer = new StringBuffer();
if (pmmlIn == null) {
return new PMMLPortObject(createPMMLSpec(null, outSpec, res));
} else {
PMMLDocument pmmldoc;
try (LockedSupplier<Document> supplier = pmmlIn.getPMMLValue().getDocumentSupplier()) {
pmmldoc = PMMLDocument.Factory.parse(supplier.get());
}
// Inspect models to check if they use any excluded columns
List<PMMLModelWrapper> models = PMMLModelWrapper.getModelListFromPMMLDocument(pmmldoc);
for (PMMLModelWrapper model : models) {
MiningSchema ms = model.getMiningSchema();
for (MiningField mf : ms.getMiningFieldList()) {
if (isExcluded(mf.getName(), res)) {
if (warningBuffer.length() != 0) {
warningBuffer.append("\n");
}
warningBuffer.append(model.getModelType().name() + " uses excluded column " + mf.getName());
}
}
}
ArrayList<String> warningFields = new ArrayList<String>();
PMML pmml = pmmldoc.getPMML();
// Now check the transformations if they exist
if (pmml.getTransformationDictionary() != null) {
for (DerivedField df : pmml.getTransformationDictionary().getDerivedFieldList()) {
FieldRef fr = df.getFieldRef();
if (fr != null && isExcluded(fr.getField(), res)) {
warningFields.add(fr.getField());
}
Aggregate a = df.getAggregate();
if (a != null && isExcluded(a.getField(), res)) {
warningFields.add(a.getField());
}
Apply ap = df.getApply();
if (ap != null) {
for (FieldRef fieldRef : ap.getFieldRefList()) {
if (isExcluded(fieldRef.getField(), res)) {
warningFields.add(fieldRef.getField());
break;
}
}
}
Discretize d = df.getDiscretize();
if (d != null && isExcluded(d.getField(), res)) {
warningFields.add(d.getField());
}
MapValues mv = df.getMapValues();
if (mv != null) {
for (FieldColumnPair fcp : mv.getFieldColumnPairList()) {
if (isExcluded(fcp.getField(), res)) {
warningFields.add(fcp.getField());
}
}
}
NormContinuous nc = df.getNormContinuous();
if (nc != null && isExcluded(nc.getField(), res)) {
warningFields.add(nc.getField());
}
NormDiscrete nd = df.getNormDiscrete();
if (nd != null && isExcluded(nd.getField(), res)) {
warningFields.add(nd.getField());
}
}
}
DataDictionary dict = pmml.getDataDictionary();
List<DataField> fields = dict.getDataFieldList();
// Apply filter to spec
int numFields = 0;
for (int i = fields.size() - 1; i >= 0; i--) {
if (isExcluded(fields.get(i).getName(), res)) {
dict.removeDataField(i);
} else {
numFields++;
}
}
dict.setNumberOfFields(new BigInteger(Integer.toString(numFields)));
pmml.setDataDictionary(dict);
pmmldoc.setPMML(pmml);
// generate warnings and set as warning message
for (String s : warningFields) {
if (warningBuffer.length() != 0) {
warningBuffer.append("\n");
}
warningBuffer.append("Transformation dictionary uses excluded column " + s);
}
if (warningBuffer.length() > 0) {
setWarningMessage(warningBuffer.toString().trim());
}
PMMLPortObject outport = null;
try {
outport = new PMMLPortObject(createPMMLSpec(pmmlIn.getSpec(), outSpec, res), pmmldoc);
} catch (IllegalArgumentException e) {
if (res.getIncludes().length == 0) {
throw new IllegalArgumentException("Excluding all columns produces invalid PMML", e);
} else {
throw e;
}
}
return outport;
}
}
Aggregations