Search in sources :

Example 1 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class RuleEngineFilter2PortsNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        private SimpleStreamableOperatorInternals m_internals;

        /**
         * {@inheritDoc}
         */
        @Override
        public void loadInternals(final StreamableOperatorInternals internals) {
            m_internals = (SimpleStreamableOperatorInternals) internals;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
            // count number of rows
            long count = 0;
            RowInput rowInput = (RowInput) inputs[DATA_PORT];
            while (rowInput.poll() != null) {
                count++;
            }
            m_internals.getConfig().addLong(CFG_ROW_COUNT, count);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public StreamableOperatorInternals saveInternals() {
            return m_internals;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            long rowCount = -1L;
            if (m_internals.getConfig().containsKey(CFG_ROW_COUNT)) {
                rowCount = m_internals.getConfig().getLong(CFG_ROW_COUNT);
            }
            m_rulesList.clear();
            final PortInput rulePort = inputs[RULE_PORT];
            if (rulePort instanceof PortObjectInput) {
                PortObjectInput poRule = (PortObjectInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules((BufferedDataTable) poRule.getPortObject(), m_settings, RuleNodeSettings.RuleFilter));
            } else if (rulePort instanceof RowInput) {
                RowInput riRule = (RowInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules(riRule, m_settings, RuleNodeSettings.RuleFilter));
            }
            final DataTableSpec spec = (DataTableSpec) inSpecs[DATA_PORT];
            try {
                parseRules(spec, RuleNodeSettings.RuleSplitter);
            } catch (final ParseException e) {
                throw new InvalidSettingsException(e);
            }
            final RowInput inputPartitions = (RowInput) inputs[DATA_PORT];
            final List<Rule> rules = parseRules(inputPartitions.getDataTableSpec(), RuleNodeSettings.RuleFilter);
            final RowOutput first = (RowOutput) outputs[0];
            final int nrOutPorts = getNrOutPorts();
            final RowOutput second = nrOutPorts > 1 ? (RowOutput) outputs[1] : new RowOutput() {

                @Override
                public void push(final DataRow row) throws InterruptedException {
                // do nothing
                }

                @Override
                public void close() throws InterruptedException {
                // do nothing
                }
            };
            final RowOutput[] containers = new RowOutput[] { first, second };
            final int matchIndex = m_includeOnMatch ? 0 : 1;
            final int otherIndex = 1 - matchIndex;
            try {
                final MutableLong rowIdx = new MutableLong(0L);
                final long rows = rowCount;
                final VariableProvider provider = new VariableProvider() {

                    @Override
                    public Object readVariable(final String name, final Class<?> type) {
                        return RuleEngineFilter2PortsNodeModel.this.readVariable(name, type);
                    }

                    @Override
                    @Deprecated
                    public int getRowCount() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowCountLong() {
                        return rows;
                    }

                    @Override
                    @Deprecated
                    public int getRowIndex() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowIndexLong() {
                        return rowIdx.longValue();
                    }
                };
                DataRow row;
                while ((row = inputPartitions.poll()) != null) {
                    rowIdx.increment();
                    if (rows > 0) {
                        exec.setProgress(rowIdx.longValue() / (double) rows, () -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    } else {
                        exec.setMessage(() -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    }
                    exec.checkCanceled();
                    boolean wasMatch = false;
                    for (Rule r : rules) {
                        if (r.getCondition().matches(row, provider).getOutcome() == MatchState.matchedAndStop) {
                            // r.getSideEffect().perform(row, provider);
                            DataValue value = r.getOutcome().getComputedResult(row, provider);
                            if (value instanceof BooleanValue) {
                                final BooleanValue bv = (BooleanValue) value;
                                containers[bv.getBooleanValue() ? matchIndex : otherIndex].push(row);
                            } else {
                                containers[matchIndex].push(row);
                            }
                            wasMatch = true;
                            break;
                        }
                    }
                    if (!wasMatch) {
                        containers[otherIndex].push(row);
                    }
                }
            } finally {
                try {
                    second.close();
                } finally {
                    first.close();
                }
            }
        }
    };
}
Also used : DataTableSpec(org.knime.core.data.DataTableSpec) DataValue(org.knime.core.data.DataValue) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) StreamableOperatorInternals(org.knime.core.node.streamable.StreamableOperatorInternals) SimpleStreamableOperatorInternals(org.knime.core.node.streamable.simple.SimpleStreamableOperatorInternals) DataTableRowInput(org.knime.core.node.streamable.DataTableRowInput) RowInput(org.knime.core.node.streamable.RowInput) DataRow(org.knime.core.data.DataRow) PortObjectInput(org.knime.core.node.streamable.PortObjectInput) RowAppenderRowOutput(org.knime.base.node.rules.engine.RowAppenderRowOutput) BufferedDataTableRowOutput(org.knime.core.node.streamable.BufferedDataTableRowOutput) RowOutput(org.knime.core.node.streamable.RowOutput) VariableProvider(org.knime.base.node.rules.engine.VariableProvider) BooleanValue(org.knime.core.data.BooleanValue) BufferedDataTable(org.knime.core.node.BufferedDataTable) PortInput(org.knime.core.node.streamable.PortInput) SimpleStreamableOperatorInternals(org.knime.core.node.streamable.simple.SimpleStreamableOperatorInternals) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ExecutionContext(org.knime.core.node.ExecutionContext) InvalidSettingsException(org.knime.core.node.InvalidSettingsException) ParseException(java.text.ParseException) Rule(org.knime.base.node.rules.engine.Rule)

Example 2 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method loadNominalHistogramsPrivate.

private static Map<Integer, HistogramNominalModel> loadNominalHistogramsPrivate(final File histogramsGz, final int[] nominalKeysSize) throws IOException, InvalidSettingsException {
    final FileInputStream is = new FileInputStream(histogramsGz);
    final GZIPInputStream inData = new GZIPInputStream(is);
    final ConfigRO config = NodeSettings.loadFromXML(inData);
    Map<Integer, HistogramNominalModel> histograms = new HashMap<Integer, HistogramNominalModel>();
    // .getConfig(HISTOGRAMS);
    ConfigRO hs = config;
    int[] nomColumnIndices = config.getIntArray(NOMINAL_COLUMNS);
    for (int colIdx : nomColumnIndices) {
        Config h = hs.getConfig(HISTOGRAM + colIdx);
        int maxCount = h.getInt(MAX_COUNT);
        int rowCount = h.getInt(ROW_COUNT);
        String colName = h.getString(COL_NAME);
        String[] values = h.getStringArray(BIN_VALUES);
        int[] binCounts = h.getIntArray(BIN_COUNTS);
        Map<DataValue, Integer> bins = new HashMap<DataValue, Integer>();
        for (int i = binCounts.length; i-- > 0; ) {
            if (values[i] == "?") {
                bins.put(new MissingCell(null), binCounts[i]);
            } else {
                bins.put(new StringCell(values[i]), binCounts[i]);
            }
        }
        HistogramNominalModel histogramData = new HistogramNominalModel(bins, colIdx, colName, rowCount);
        histogramData.setMaxCount(maxCount);
        histogramData.setRowCount(rowCount);
        // assert Math.abs(histogramData.m_width - width) < 1e-9: "histogram data width: " + histogramData.m_width + " width: " + width;
        assert nominalKeysSize[colIdx] == bins.size() : "Saved size of nominal bins: " + nominalKeysSize[colIdx] + ", restored from the file: " + bins.size();
        histograms.put(colIdx, histogramData);
    }
    return histograms;
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) DataValue(org.knime.core.data.DataValue) Config(org.knime.core.node.config.Config) FileInputStream(java.io.FileInputStream) GZIPInputStream(java.util.zip.GZIPInputStream) MissingCell(org.knime.core.data.MissingCell) StringCell(org.knime.core.data.def.StringCell) ConfigRO(org.knime.core.node.config.ConfigRO)

Example 3 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method constructFromDataArray.

/**
 * Constructs the helper data structures from the numeric hostigran models and the data as {@link DataArray}.
 *
 * @param histograms The numeric histograms.
 * @param data The input data.
 * @param nominalColumnNames The nominal column names.
 * @return The helper data structures.
 * @see #construct(Map, DataTable, Set)
 */
protected static Pair<Map<Integer, Map<Integer, Set<RowKey>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> constructFromDataArray(final Map<Integer, HistogramNumericModel> histograms, final DataTable data, final Set<String> nominalColumnNames) {
    Map<Integer, Map<Integer, Set<RowKey>>> numericMapping = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
    Map<Integer, Map<DataValue, Set<RowKey>>> nominalMapping = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
    DataTableSpec tableSpec = data.getDataTableSpec();
    for (DataColumnSpec colSpec : tableSpec) {
        int colIndex = tableSpec.findColumnIndex(colSpec.getName());
        if (colSpec.getType().isCompatible(DoubleValue.class)) {
            // + colIndex;
            if (histograms.containsKey(Integer.valueOf(colIndex)) && histograms.get(colIndex) != null) {
                numericMapping.put(colIndex, new HashMap<Integer, Set<RowKey>>());
            }
        }
        if (colSpec.getDomain().hasValues() || nominalColumnNames.contains(colSpec.getName())) {
            nominalMapping.put(colIndex, new HashMap<DataValue, Set<RowKey>>());
        }
    }
    for (DataRow dataRow : data) {
        for (Entry<Integer, Map<Integer, Set<RowKey>>> outer : numericMapping.entrySet()) {
            Integer key = outer.getKey();
            DataCell cell = dataRow.getCell(key);
            if (cell instanceof DoubleValue) {
                DoubleValue dv = (DoubleValue) cell;
                Integer bin = Integer.valueOf(histograms.get(key).findBin(dv));
                Map<Integer, Set<RowKey>> inner = outer.getValue();
                if (!inner.containsKey(bin)) {
                    inner.put(bin, new HashSet<RowKey>());
                }
                inner.get(bin).add(dataRow.getKey());
            }
        }
        for (Entry<Integer, Map<DataValue, Set<RowKey>>> outer : nominalMapping.entrySet()) {
            int key = outer.getKey().intValue();
            DataCell cell = dataRow.getCell(key);
            if (!cell.isMissing()) /* && cell instanceof NominalValue*/
            {
                Map<DataValue, Set<RowKey>> inner = outer.getValue();
                if (!inner.containsKey(cell)) {
                    inner.put(cell, new HashSet<RowKey>());
                }
                inner.get(cell).add(dataRow.getKey());
            }
        }
    }
    return Pair.create(numericMapping, nominalMapping);
}
Also used : DataTableSpec(org.knime.core.data.DataTableSpec) Set(java.util.Set) HashSet(java.util.HashSet) RowKey(org.knime.core.data.RowKey) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) DataValue(org.knime.core.data.DataValue) DataRow(org.knime.core.data.DataRow) DataColumnSpec(org.knime.core.data.DataColumnSpec) DoubleValue(org.knime.core.data.DoubleValue) DataCell(org.knime.core.data.DataCell) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap)

Example 4 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method loadHistograms.

/**
 * Loads the histograms from the saved internal files.
 *
 * @param histogramsGz The file for the histograms.
 * @param dataArrayGz The data array file for the row keys.
 * @param nominalColumns The nominal columns.
 * @param strategy The strategy used to compute the bins.
 * @param means The mean values for the numeric columns.
 * @return A triple (Pair(Pair(,),)) of histograms, numeric and nominal row keys.
 * @throws IOException Failed to read the files.
 * @throws InvalidSettingsException Something went wrong.
 */
public static Pair<Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> loadHistograms(final File histogramsGz, final File dataArrayGz, final Set<String> nominalColumns, final BinNumberSelectionStrategy strategy, final double[] means) throws IOException, InvalidSettingsException {
    Map<Integer, Map<Integer, Set<RowKey>>> numericKeys = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
    Map<Integer, HistogramNumericModel> histograms = loadHistogramsPrivate(histogramsGz, numericKeys, strategy, means);
    Map<Integer, Map<DataValue, Set<RowKey>>> nominalKeys = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
    ContainerTable table = DataContainer.readFromZip(dataArrayGz);
    Set<Integer> numericColIndices = numericKeys.keySet();
    for (String colName : nominalColumns) {
        int colIndex = table.getDataTableSpec().findColumnIndex(colName);
        if (colIndex < 0) {
            continue;
        }
        nominalKeys.put(Integer.valueOf(colIndex), new HashMap<DataValue, Set<RowKey>>());
    }
    for (DataRow dataRow : table) {
        for (Integer col : numericColIndices) {
            // Integer col = Integer.valueOf(colIdx);
            HistogramNumericModel hd = histograms.get(col);
            Map<Integer, Set<RowKey>> map = numericKeys.get(col);
            DataCell cell = dataRow.getCell(col.intValue());
            if (!cell.isMissing() && cell instanceof DoubleValue) {
                DoubleValue dv = (DoubleValue) cell;
                Integer bin = Integer.valueOf(hd.findBin(dv));
                if (!map.containsKey(bin)) {
                    map.put(bin, new HashSet<RowKey>());
                }
                map.get(bin).add(dataRow.getKey());
            }
        }
        for (Entry<Integer, Map<DataValue, Set<RowKey>>> entry : nominalKeys.entrySet()) {
            DataCell value = dataRow.getCell(entry.getKey().intValue());
            Map<DataValue, Set<RowKey>> map = entry.getValue();
            if (!map.containsKey(value)) {
                map.put(value, new HashSet<RowKey>());
            }
            map.get(value).add(dataRow.getKey());
        }
    }
    return Pair.create(new Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>(histograms, numericKeys), nominalKeys);
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) RowKey(org.knime.core.data.RowKey) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) DataValue(org.knime.core.data.DataValue) DataRow(org.knime.core.data.DataRow) ContainerTable(org.knime.core.data.container.ContainerTable) DoubleValue(org.knime.core.data.DoubleValue) DataCell(org.knime.core.data.DataCell) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap)

Example 5 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class DateTimeDifferenceNodeDialog method updateDateTimeComponents.

@SuppressWarnings("unchecked")
private void updateDateTimeComponents(final SettingsModelDateTime dateTimeModel) {
    if (m_dialogComp1stColSelect.getSelectedAsSpec() != null) {
        final DataType type = m_dialogComp1stColSelect.getSelectedAsSpec().getType();
        // adapt column filter of column selection for 2nd column
        try {
            final List<Class<? extends DataValue>> valueClasses = type.getValueClasses();
            if (valueClasses.contains(ZonedDateTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(ZonedDateTimeValue.class));
            } else if (valueClasses.contains(LocalDateTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateTimeValue.class));
            } else if (valueClasses.contains(LocalDateValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateValue.class));
            } else if (valueClasses.contains(LocalTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalTimeValue.class));
            }
        } catch (NotConfigurableException ex) {
        // will never happen, because there is always at least the selected 1st column available
        }
        // adapt date&time component
        dateTimeModel.setUseDate(!type.isCompatible(LocalTimeValue.class));
        dateTimeModel.setUseTime(!type.isCompatible(LocalDateValue.class));
        dateTimeModel.setUseZone(type.isCompatible(ZonedDateTimeValue.class));
    }
    updateWarningLabel();
}
Also used : NotConfigurableException(org.knime.core.node.NotConfigurableException) LocalTimeValue(org.knime.core.data.time.localtime.LocalTimeValue) DataValue(org.knime.core.data.DataValue) DataType(org.knime.core.data.DataType) LocalDateValue(org.knime.core.data.time.localdate.LocalDateValue) DataValueColumnFilter(org.knime.core.node.util.DataValueColumnFilter) ZonedDateTimeValue(org.knime.core.data.time.zoneddatetime.ZonedDateTimeValue) LocalDateTimeValue(org.knime.core.data.time.localdatetime.LocalDateTimeValue)

Aggregations

DataValue (org.knime.core.data.DataValue)28 DataCell (org.knime.core.data.DataCell)9 DoubleValue (org.knime.core.data.DoubleValue)8 ArrayList (java.util.ArrayList)7 HashMap (java.util.HashMap)7 DataType (org.knime.core.data.DataType)6 HashSet (java.util.HashSet)5 Map (java.util.Map)5 DataTableSpec (org.knime.core.data.DataTableSpec)5 StringCell (org.knime.core.data.def.StringCell)5 LinkedHashMap (java.util.LinkedHashMap)4 LinkedHashSet (java.util.LinkedHashSet)4 Set (java.util.Set)4 DataColumnSpec (org.knime.core.data.DataColumnSpec)4 DataRow (org.knime.core.data.DataRow)3 DefaultRow (org.knime.core.data.def.DefaultRow)3 ActionEvent (java.awt.event.ActionEvent)2 ActionListener (java.awt.event.ActionListener)2 FileInputStream (java.io.FileInputStream)2 JMenu (javax.swing.JMenu)2