Examples with DataValue org.knime.core.data.DataValue used on opensource projects

Example 1 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class RuleEngineFilter2PortsNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        private SimpleStreamableOperatorInternals m_internals;

        /**
         * {@inheritDoc}
         */
        @Override
        public void loadInternals(final StreamableOperatorInternals internals) {
            m_internals = (SimpleStreamableOperatorInternals) internals;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
            // count number of rows
            long count = 0;
            RowInput rowInput = (RowInput) inputs[DATA_PORT];
            while (rowInput.poll() != null) {
                count++;
            }
            m_internals.getConfig().addLong(CFG_ROW_COUNT, count);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public StreamableOperatorInternals saveInternals() {
            return m_internals;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            long rowCount = -1L;
            if (m_internals.getConfig().containsKey(CFG_ROW_COUNT)) {
                rowCount = m_internals.getConfig().getLong(CFG_ROW_COUNT);
            }
            m_rulesList.clear();
            final PortInput rulePort = inputs[RULE_PORT];
            if (rulePort instanceof PortObjectInput) {
                PortObjectInput poRule = (PortObjectInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules((BufferedDataTable) poRule.getPortObject(), m_settings, RuleNodeSettings.RuleFilter));
            } else if (rulePort instanceof RowInput) {
                RowInput riRule = (RowInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules(riRule, m_settings, RuleNodeSettings.RuleFilter));
            }
            final DataTableSpec spec = (DataTableSpec) inSpecs[DATA_PORT];
            try {
                parseRules(spec, RuleNodeSettings.RuleSplitter);
            } catch (final ParseException e) {
                throw new InvalidSettingsException(e);
            }
            final RowInput inputPartitions = (RowInput) inputs[DATA_PORT];
            final List<Rule> rules = parseRules(inputPartitions.getDataTableSpec(), RuleNodeSettings.RuleFilter);
            final RowOutput first = (RowOutput) outputs[0];
            final int nrOutPorts = getNrOutPorts();
            final RowOutput second = nrOutPorts > 1 ? (RowOutput) outputs[1] : new RowOutput() {

                @Override
                public void push(final DataRow row) throws InterruptedException {
                // do nothing
                }

                @Override
                public void close() throws InterruptedException {
                // do nothing
                }
            };
            final RowOutput[] containers = new RowOutput[] { first, second };
            final int matchIndex = m_includeOnMatch ? 0 : 1;
            final int otherIndex = 1 - matchIndex;
            try {
                final MutableLong rowIdx = new MutableLong(0L);
                final long rows = rowCount;
                final VariableProvider provider = new VariableProvider() {

                    @Override
                    public Object readVariable(final String name, final Class<?> type) {
                        return RuleEngineFilter2PortsNodeModel.this.readVariable(name, type);
                    }

                    @Override
                    @Deprecated
                    public int getRowCount() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowCountLong() {
                        return rows;
                    }

                    @Override
                    @Deprecated
                    public int getRowIndex() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowIndexLong() {
                        return rowIdx.longValue();
                    }
                };
                DataRow row;
                while ((row = inputPartitions.poll()) != null) {
                    rowIdx.increment();
                    if (rows > 0) {
                        exec.setProgress(rowIdx.longValue() / (double) rows, () -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    } else {
                        exec.setMessage(() -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    }
                    exec.checkCanceled();
                    boolean wasMatch = false;
                    for (Rule r : rules) {
                        if (r.getCondition().matches(row, provider).getOutcome() == MatchState.matchedAndStop) {
                            // r.getSideEffect().perform(row, provider);
                            DataValue value = r.getOutcome().getComputedResult(row, provider);
                            if (value instanceof BooleanValue) {
                                final BooleanValue bv = (BooleanValue) value;
                                containers[bv.getBooleanValue() ? matchIndex : otherIndex].push(row);
                            } else {
                                containers[matchIndex].push(row);
                            }
                            wasMatch = true;
                            break;
                        }
                    }
                    if (!wasMatch) {
                        containers[otherIndex].push(row);
                    }
                }
            } finally {
                try {
                    second.close();
                } finally {
                    first.close();
                }
            }
        }
    };
}

Example 2 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method loadNominalHistogramsPrivate.

private static Map<Integer, HistogramNominalModel> loadNominalHistogramsPrivate(final File histogramsGz, final int[] nominalKeysSize) throws IOException, InvalidSettingsException {
    final FileInputStream is = new FileInputStream(histogramsGz);
    final GZIPInputStream inData = new GZIPInputStream(is);
    final ConfigRO config = NodeSettings.loadFromXML(inData);
    Map<Integer, HistogramNominalModel> histograms = new HashMap<Integer, HistogramNominalModel>();
    // .getConfig(HISTOGRAMS);
    ConfigRO hs = config;
    int[] nomColumnIndices = config.getIntArray(NOMINAL_COLUMNS);
    for (int colIdx : nomColumnIndices) {
        Config h = hs.getConfig(HISTOGRAM + colIdx);
        int maxCount = h.getInt(MAX_COUNT);
        int rowCount = h.getInt(ROW_COUNT);
        String colName = h.getString(COL_NAME);
        String[] values = h.getStringArray(BIN_VALUES);
        int[] binCounts = h.getIntArray(BIN_COUNTS);
        Map<DataValue, Integer> bins = new HashMap<DataValue, Integer>();
        for (int i = binCounts.length; i-- > 0; ) {
            if (values[i] == "?") {
                bins.put(new MissingCell(null), binCounts[i]);
            } else {
                bins.put(new StringCell(values[i]), binCounts[i]);
            }
        }
        HistogramNominalModel histogramData = new HistogramNominalModel(bins, colIdx, colName, rowCount);
        histogramData.setMaxCount(maxCount);
        histogramData.setRowCount(rowCount);
        // assert Math.abs(histogramData.m_width - width) < 1e-9: "histogram data width: " + histogramData.m_width + " width: " + width;
        assert nominalKeysSize[colIdx] == bins.size() : "Saved size of nominal bins: " + nominalKeysSize[colIdx] + ", restored from the file: " + bins.size();
        histograms.put(colIdx, histogramData);
    }
    return histograms;
}

Example 3 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method constructFromDataArray.

/**
 * Constructs the helper data structures from the numeric hostigran models and the data as {@link DataArray}.
 *
 * @param histograms The numeric histograms.
 * @param data The input data.
 * @param nominalColumnNames The nominal column names.
 * @return The helper data structures.
 * @see #construct(Map, DataTable, Set)
 */
protected static Pair<Map<Integer, Map<Integer, Set<RowKey>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> constructFromDataArray(final Map<Integer, HistogramNumericModel> histograms, final DataTable data, final Set<String> nominalColumnNames) {
    Map<Integer, Map<Integer, Set<RowKey>>> numericMapping = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
    Map<Integer, Map<DataValue, Set<RowKey>>> nominalMapping = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
    DataTableSpec tableSpec = data.getDataTableSpec();
    for (DataColumnSpec colSpec : tableSpec) {
        int colIndex = tableSpec.findColumnIndex(colSpec.getName());
        if (colSpec.getType().isCompatible(DoubleValue.class)) {
            // + colIndex;
            if (histograms.containsKey(Integer.valueOf(colIndex)) && histograms.get(colIndex) != null) {
                numericMapping.put(colIndex, new HashMap<Integer, Set<RowKey>>());
            }
        }
        if (colSpec.getDomain().hasValues() || nominalColumnNames.contains(colSpec.getName())) {
            nominalMapping.put(colIndex, new HashMap<DataValue, Set<RowKey>>());
        }
    }
    for (DataRow dataRow : data) {
        for (Entry<Integer, Map<Integer, Set<RowKey>>> outer : numericMapping.entrySet()) {
            Integer key = outer.getKey();
            DataCell cell = dataRow.getCell(key);
            if (cell instanceof DoubleValue) {
                DoubleValue dv = (DoubleValue) cell;
                Integer bin = Integer.valueOf(histograms.get(key).findBin(dv));
                Map<Integer, Set<RowKey>> inner = outer.getValue();
                if (!inner.containsKey(bin)) {
                    inner.put(bin, new HashSet<RowKey>());
                }
                inner.get(bin).add(dataRow.getKey());
            }
        }
        for (Entry<Integer, Map<DataValue, Set<RowKey>>> outer : nominalMapping.entrySet()) {
            int key = outer.getKey().intValue();
            DataCell cell = dataRow.getCell(key);
            if (!cell.isMissing()) /* && cell instanceof NominalValue*/
            {
                Map<DataValue, Set<RowKey>> inner = outer.getValue();
                if (!inner.containsKey(cell)) {
                    inner.put(cell, new HashSet<RowKey>());
                }
                inner.get(cell).add(dataRow.getKey());
            }
        }
    }
    return Pair.create(numericMapping, nominalMapping);
}

Example 4 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class HistogramColumn method loadHistograms.

/**
 * Loads the histograms from the saved internal files.
 *
 * @param histogramsGz The file for the histograms.
 * @param dataArrayGz The data array file for the row keys.
 * @param nominalColumns The nominal columns.
 * @param strategy The strategy used to compute the bins.
 * @param means The mean values for the numeric columns.
 * @return A triple (Pair(Pair(,),)) of histograms, numeric and nominal row keys.
 * @throws IOException Failed to read the files.
 * @throws InvalidSettingsException Something went wrong.
 */
public static Pair<Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> loadHistograms(final File histogramsGz, final File dataArrayGz, final Set<String> nominalColumns, final BinNumberSelectionStrategy strategy, final double[] means) throws IOException, InvalidSettingsException {
    Map<Integer, Map<Integer, Set<RowKey>>> numericKeys = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
    Map<Integer, HistogramNumericModel> histograms = loadHistogramsPrivate(histogramsGz, numericKeys, strategy, means);
    Map<Integer, Map<DataValue, Set<RowKey>>> nominalKeys = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
    ContainerTable table = DataContainer.readFromZip(dataArrayGz);
    Set<Integer> numericColIndices = numericKeys.keySet();
    for (String colName : nominalColumns) {
        int colIndex = table.getDataTableSpec().findColumnIndex(colName);
        if (colIndex < 0) {
            continue;
        }
        nominalKeys.put(Integer.valueOf(colIndex), new HashMap<DataValue, Set<RowKey>>());
    }
    for (DataRow dataRow : table) {
        for (Integer col : numericColIndices) {
            // Integer col = Integer.valueOf(colIdx);
            HistogramNumericModel hd = histograms.get(col);
            Map<Integer, Set<RowKey>> map = numericKeys.get(col);
            DataCell cell = dataRow.getCell(col.intValue());
            if (!cell.isMissing() && cell instanceof DoubleValue) {
                DoubleValue dv = (DoubleValue) cell;
                Integer bin = Integer.valueOf(hd.findBin(dv));
                if (!map.containsKey(bin)) {
                    map.put(bin, new HashSet<RowKey>());
                }
                map.get(bin).add(dataRow.getKey());
            }
        }
        for (Entry<Integer, Map<DataValue, Set<RowKey>>> entry : nominalKeys.entrySet()) {
            DataCell value = dataRow.getCell(entry.getKey().intValue());
            Map<DataValue, Set<RowKey>> map = entry.getValue();
            if (!map.containsKey(value)) {
                map.put(value, new HashSet<RowKey>());
            }
            map.get(value).add(dataRow.getKey());
        }
    }
    return Pair.create(new Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>(histograms, numericKeys), nominalKeys);
}

Example 5 with DataValue

use of org.knime.core.data.DataValue in project knime-core by knime.

the class DateTimeDifferenceNodeDialog method updateDateTimeComponents.

@SuppressWarnings("unchecked")
private void updateDateTimeComponents(final SettingsModelDateTime dateTimeModel) {
    if (m_dialogComp1stColSelect.getSelectedAsSpec() != null) {
        final DataType type = m_dialogComp1stColSelect.getSelectedAsSpec().getType();
        // adapt column filter of column selection for 2nd column
        try {
            final List<Class<? extends DataValue>> valueClasses = type.getValueClasses();
            if (valueClasses.contains(ZonedDateTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(ZonedDateTimeValue.class));
            } else if (valueClasses.contains(LocalDateTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateTimeValue.class));
            } else if (valueClasses.contains(LocalDateValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateValue.class));
            } else if (valueClasses.contains(LocalTimeValue.class)) {
                m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalTimeValue.class));
            }
        } catch (NotConfigurableException ex) {
        // will never happen, because there is always at least the selected 1st column available
        }
        // adapt date&time component
        dateTimeModel.setUseDate(!type.isCompatible(LocalTimeValue.class));
        dateTimeModel.setUseTime(!type.isCompatible(LocalDateValue.class));
        dateTimeModel.setUseZone(type.isCompatible(ZonedDateTimeValue.class));
    }
    updateWarningLabel();
}