Search in sources :

Example 1 with RowOutput

use of org.knime.core.node.streamable.RowOutput in project knime-core by knime.

the class RuleEngineFilter2PortsNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        private SimpleStreamableOperatorInternals m_internals;

        /**
         * {@inheritDoc}
         */
        @Override
        public void loadInternals(final StreamableOperatorInternals internals) {
            m_internals = (SimpleStreamableOperatorInternals) internals;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
            // count number of rows
            long count = 0;
            RowInput rowInput = (RowInput) inputs[DATA_PORT];
            while (rowInput.poll() != null) {
                count++;
            }
            m_internals.getConfig().addLong(CFG_ROW_COUNT, count);
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public StreamableOperatorInternals saveInternals() {
            return m_internals;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            long rowCount = -1L;
            if (m_internals.getConfig().containsKey(CFG_ROW_COUNT)) {
                rowCount = m_internals.getConfig().getLong(CFG_ROW_COUNT);
            }
            m_rulesList.clear();
            final PortInput rulePort = inputs[RULE_PORT];
            if (rulePort instanceof PortObjectInput) {
                PortObjectInput poRule = (PortObjectInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules((BufferedDataTable) poRule.getPortObject(), m_settings, RuleNodeSettings.RuleFilter));
            } else if (rulePort instanceof RowInput) {
                RowInput riRule = (RowInput) rulePort;
                m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules(riRule, m_settings, RuleNodeSettings.RuleFilter));
            }
            final DataTableSpec spec = (DataTableSpec) inSpecs[DATA_PORT];
            try {
                parseRules(spec, RuleNodeSettings.RuleSplitter);
            } catch (final ParseException e) {
                throw new InvalidSettingsException(e);
            }
            final RowInput inputPartitions = (RowInput) inputs[DATA_PORT];
            final List<Rule> rules = parseRules(inputPartitions.getDataTableSpec(), RuleNodeSettings.RuleFilter);
            final RowOutput first = (RowOutput) outputs[0];
            final int nrOutPorts = getNrOutPorts();
            final RowOutput second = nrOutPorts > 1 ? (RowOutput) outputs[1] : new RowOutput() {

                @Override
                public void push(final DataRow row) throws InterruptedException {
                // do nothing
                }

                @Override
                public void close() throws InterruptedException {
                // do nothing
                }
            };
            final RowOutput[] containers = new RowOutput[] { first, second };
            final int matchIndex = m_includeOnMatch ? 0 : 1;
            final int otherIndex = 1 - matchIndex;
            try {
                final MutableLong rowIdx = new MutableLong(0L);
                final long rows = rowCount;
                final VariableProvider provider = new VariableProvider() {

                    @Override
                    public Object readVariable(final String name, final Class<?> type) {
                        return RuleEngineFilter2PortsNodeModel.this.readVariable(name, type);
                    }

                    @Override
                    @Deprecated
                    public int getRowCount() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowCountLong() {
                        return rows;
                    }

                    @Override
                    @Deprecated
                    public int getRowIndex() {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public long getRowIndexLong() {
                        return rowIdx.longValue();
                    }
                };
                DataRow row;
                while ((row = inputPartitions.poll()) != null) {
                    rowIdx.increment();
                    if (rows > 0) {
                        exec.setProgress(rowIdx.longValue() / (double) rows, () -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    } else {
                        exec.setMessage(() -> "Adding row " + rowIdx.longValue() + " of " + rows);
                    }
                    exec.checkCanceled();
                    boolean wasMatch = false;
                    for (Rule r : rules) {
                        if (r.getCondition().matches(row, provider).getOutcome() == MatchState.matchedAndStop) {
                            // r.getSideEffect().perform(row, provider);
                            DataValue value = r.getOutcome().getComputedResult(row, provider);
                            if (value instanceof BooleanValue) {
                                final BooleanValue bv = (BooleanValue) value;
                                containers[bv.getBooleanValue() ? matchIndex : otherIndex].push(row);
                            } else {
                                containers[matchIndex].push(row);
                            }
                            wasMatch = true;
                            break;
                        }
                    }
                    if (!wasMatch) {
                        containers[otherIndex].push(row);
                    }
                }
            } finally {
                try {
                    second.close();
                } finally {
                    first.close();
                }
            }
        }
    };
}
Also used : DataTableSpec(org.knime.core.data.DataTableSpec) DataValue(org.knime.core.data.DataValue) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) StreamableOperatorInternals(org.knime.core.node.streamable.StreamableOperatorInternals) SimpleStreamableOperatorInternals(org.knime.core.node.streamable.simple.SimpleStreamableOperatorInternals) DataTableRowInput(org.knime.core.node.streamable.DataTableRowInput) RowInput(org.knime.core.node.streamable.RowInput) DataRow(org.knime.core.data.DataRow) PortObjectInput(org.knime.core.node.streamable.PortObjectInput) RowAppenderRowOutput(org.knime.base.node.rules.engine.RowAppenderRowOutput) BufferedDataTableRowOutput(org.knime.core.node.streamable.BufferedDataTableRowOutput) RowOutput(org.knime.core.node.streamable.RowOutput) VariableProvider(org.knime.base.node.rules.engine.VariableProvider) BooleanValue(org.knime.core.data.BooleanValue) BufferedDataTable(org.knime.core.node.BufferedDataTable) PortInput(org.knime.core.node.streamable.PortInput) SimpleStreamableOperatorInternals(org.knime.core.node.streamable.simple.SimpleStreamableOperatorInternals) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ExecutionContext(org.knime.core.node.ExecutionContext) InvalidSettingsException(org.knime.core.node.InvalidSettingsException) ParseException(java.text.ParseException) Rule(org.knime.base.node.rules.engine.Rule)

Example 2 with RowOutput

use of org.knime.core.node.streamable.RowOutput in project knime-core by knime.

the class NewToOldTimeNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public StreamableOperatorInternals saveInternals() {
            return null;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            final RowInput in = (RowInput) inputs[0];
            final RowOutput out = (RowOutput) outputs[0];
            final DataTableSpec inSpec = in.getDataTableSpec();
            String[] includeList = m_colSelect.applyTo(inSpec).getIncludes();
            final int[] includeIndeces = Arrays.stream(includeList).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
            DataRow row;
            while ((row = in.poll()) != null) {
                exec.checkCanceled();
                DataCell[] datacells = new DataCell[includeIndeces.length];
                for (int i = 0; i < includeIndeces.length; i++) {
                    if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                        final DataColumnSpecCreator dataColumnSpecCreator = new DataColumnSpecCreator(includeList[i], DateAndTimeCell.TYPE);
                        final ConvertTimeCellFactory cellFac = new ConvertTimeCellFactory(dataColumnSpecCreator.createSpec(), includeIndeces[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    } else {
                        final DataColumnSpec dataColSpec = new UniqueNameGenerator(inSpec).newColumn(includeList[i] + m_suffix.getStringValue(), DateAndTimeCell.TYPE);
                        final ConvertTimeCellFactory cellFac = new ConvertTimeCellFactory(dataColSpec, includeIndeces[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    }
                }
                if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                    out.push(new ReplacedColumnsDataRow(row, datacells, includeIndeces));
                } else {
                    out.push(new AppendedColumnRow(row, datacells));
                }
            }
            in.close();
            out.close();
        }
    };
}
Also used : Arrays(java.util.Arrays) NodeSettingsRO(org.knime.core.node.NodeSettingsRO) DataTableSpec(org.knime.core.data.DataTableSpec) InvalidSettingsException(org.knime.core.node.InvalidSettingsException) CanceledExecutionException(org.knime.core.node.CanceledExecutionException) ZonedDateTime(java.time.ZonedDateTime) UniqueNameGenerator(org.knime.core.util.UniqueNameGenerator) LocalDateTime(java.time.LocalDateTime) AppendedColumnRow(org.knime.core.data.append.AppendedColumnRow) LocalTimeValue(org.knime.core.data.time.localtime.LocalTimeValue) LocalDateValue(org.knime.core.data.time.localdate.LocalDateValue) ExecutionContext(org.knime.core.node.ExecutionContext) SingleCellFactory(org.knime.core.data.container.SingleCellFactory) StreamableOperatorInternals(org.knime.core.node.streamable.StreamableOperatorInternals) DataColumnSpec(org.knime.core.data.DataColumnSpec) DataColumnSpecCreator(org.knime.core.data.DataColumnSpecCreator) LocalTime(java.time.LocalTime) ZoneOffset(java.time.ZoneOffset) DataCell(org.knime.core.data.DataCell) ZonedDateTimeValue(org.knime.core.data.time.zoneddatetime.ZonedDateTimeValue) PortInput(org.knime.core.node.streamable.PortInput) ExecutionMonitor(org.knime.core.node.ExecutionMonitor) PartitionInfo(org.knime.core.node.streamable.PartitionInfo) RowInput(org.knime.core.node.streamable.RowInput) PortObjectSpec(org.knime.core.node.port.PortObjectSpec) IOException(java.io.IOException) SettingsModelColumnFilter2(org.knime.core.node.defaultnodesettings.SettingsModelColumnFilter2) OutputPortRole(org.knime.core.node.streamable.OutputPortRole) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) NodeModel(org.knime.core.node.NodeModel) File(java.io.File) DataRow(org.knime.core.data.DataRow) SettingsModelString(org.knime.core.node.defaultnodesettings.SettingsModelString) PortOutput(org.knime.core.node.streamable.PortOutput) NodeSettingsWO(org.knime.core.node.NodeSettingsWO) TimeUnit(java.util.concurrent.TimeUnit) BufferedDataTable(org.knime.core.node.BufferedDataTable) LocalDateTimeValue(org.knime.core.data.time.localdatetime.LocalDateTimeValue) InputPortRole(org.knime.core.node.streamable.InputPortRole) LocalDate(java.time.LocalDate) ColumnRearranger(org.knime.core.data.container.ColumnRearranger) DateAndTimeCell(org.knime.core.data.date.DateAndTimeCell) RowOutput(org.knime.core.node.streamable.RowOutput) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) DataTableSpec(org.knime.core.data.DataTableSpec) DataColumnSpecCreator(org.knime.core.data.DataColumnSpecCreator) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) SettingsModelString(org.knime.core.node.defaultnodesettings.SettingsModelString) RowInput(org.knime.core.node.streamable.RowInput) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) DataRow(org.knime.core.data.DataRow) UniqueNameGenerator(org.knime.core.util.UniqueNameGenerator) RowOutput(org.knime.core.node.streamable.RowOutput) ExecutionContext(org.knime.core.node.ExecutionContext) DataColumnSpec(org.knime.core.data.DataColumnSpec) DataCell(org.knime.core.data.DataCell) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) AppendedColumnRow(org.knime.core.data.append.AppendedColumnRow)

Example 3 with RowOutput

use of org.knime.core.node.streamable.RowOutput in project knime-core by knime.

the class DateTimeToStringNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            final RowInput in = (RowInput) inputs[0];
            final RowOutput out = (RowOutput) outputs[0];
            final DataTableSpec inSpec = in.getDataTableSpec();
            final String[] includeList = m_colSelect.applyTo(inSpec).getIncludes();
            final int[] includeIndeces = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
            final boolean isReplace = m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE);
            DataRow row;
            while ((row = in.poll()) != null) {
                exec.checkCanceled();
                DataCell[] datacells = new DataCell[includeIndeces.length];
                for (int i = 0; i < includeIndeces.length; i++) {
                    if (isReplace) {
                        final DataColumnSpecCreator dataColumnSpecCreator = new DataColumnSpecCreator(includeList[i], StringCell.TYPE);
                        final TimeToStringCellFactory cellFac = new TimeToStringCellFactory(dataColumnSpecCreator.createSpec(), includeIndeces[i]);
                        datacells[i] = cellFac.getCell(row);
                    } else {
                        final DataColumnSpec dataColSpec = new UniqueNameGenerator(inSpec).newColumn(includeList[i] + m_suffix.getStringValue(), StringCell.TYPE);
                        final TimeToStringCellFactory cellFac = new TimeToStringCellFactory(dataColSpec, includeIndeces[i]);
                        datacells[i] = cellFac.getCell(row);
                    }
                }
                if (isReplace) {
                    out.push(new ReplacedColumnsDataRow(row, datacells, includeIndeces));
                } else {
                    out.push(new AppendedColumnRow(row, datacells));
                }
            }
            in.close();
            out.close();
        }
    };
}
Also used : Arrays(java.util.Arrays) NodeSettingsRO(org.knime.core.node.NodeSettingsRO) InvalidSettingsException(org.knime.core.node.InvalidSettingsException) CanceledExecutionException(org.knime.core.node.CanceledExecutionException) UniqueNameGenerator(org.knime.core.util.UniqueNameGenerator) LocalDateValue(org.knime.core.data.time.localdate.LocalDateValue) LocalDateTimeCellFactory(org.knime.core.data.time.localdatetime.LocalDateTimeCellFactory) DataColumnSpec(org.knime.core.data.DataColumnSpec) Locale(java.util.Locale) ChangeListener(javax.swing.event.ChangeListener) ZonedDateTimeValue(org.knime.core.data.time.zoneddatetime.ZonedDateTimeValue) ChangeEvent(javax.swing.event.ChangeEvent) PortInput(org.knime.core.node.streamable.PortInput) ExecutionMonitor(org.knime.core.node.ExecutionMonitor) PartitionInfo(org.knime.core.node.streamable.PartitionInfo) Collection(java.util.Collection) Set(java.util.Set) SettingsModelColumnFilter2(org.knime.core.node.defaultnodesettings.SettingsModelColumnFilter2) NodeModel(org.knime.core.node.NodeModel) LocalDateCell(org.knime.core.data.time.localdate.LocalDateCell) BufferedDataTable(org.knime.core.node.BufferedDataTable) InputPortRole(org.knime.core.node.streamable.InputPortRole) StringHistory(org.knime.core.node.util.StringHistory) LocalDateTimeCell(org.knime.core.data.time.localdatetime.LocalDateTimeCell) RowOutput(org.knime.core.node.streamable.RowOutput) DataTableSpec(org.knime.core.data.DataTableSpec) ZonedDateTimeCell(org.knime.core.data.time.zoneddatetime.ZonedDateTimeCell) AppendedColumnRow(org.knime.core.data.append.AppendedColumnRow) LocalTimeValue(org.knime.core.data.time.localtime.LocalTimeValue) ExecutionContext(org.knime.core.node.ExecutionContext) LocaleUtils(org.apache.commons.lang3.LocaleUtils) SingleCellFactory(org.knime.core.data.container.SingleCellFactory) ZonedDateTimeCellFactory(org.knime.core.data.time.zoneddatetime.ZonedDateTimeCellFactory) DataColumnSpecCreator(org.knime.core.data.DataColumnSpecCreator) DataCell(org.knime.core.data.DataCell) LinkedHashSet(java.util.LinkedHashSet) LocalDateCellFactory(org.knime.core.data.time.localdate.LocalDateCellFactory) LocalTimeCellFactory(org.knime.core.data.time.localtime.LocalTimeCellFactory) StringCellFactory(org.knime.core.data.def.StringCell.StringCellFactory) RowInput(org.knime.core.node.streamable.RowInput) LocalTimeCell(org.knime.core.data.time.localtime.LocalTimeCell) SettingsModelBoolean(org.knime.core.node.defaultnodesettings.SettingsModelBoolean) PortObjectSpec(org.knime.core.node.port.PortObjectSpec) IOException(java.io.IOException) OutputPortRole(org.knime.core.node.streamable.OutputPortRole) UnsupportedTemporalTypeException(java.time.temporal.UnsupportedTemporalTypeException) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) File(java.io.File) DataRow(org.knime.core.data.DataRow) SettingsModelString(org.knime.core.node.defaultnodesettings.SettingsModelString) PortOutput(org.knime.core.node.streamable.PortOutput) NodeSettingsWO(org.knime.core.node.NodeSettingsWO) LocalDateTimeValue(org.knime.core.data.time.localdatetime.LocalDateTimeValue) MissingCell(org.knime.core.data.MissingCell) DateTimeFormatter(java.time.format.DateTimeFormatter) ColumnRearranger(org.knime.core.data.container.ColumnRearranger) StringCell(org.knime.core.data.def.StringCell) DataType(org.knime.core.data.DataType) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) DataTableSpec(org.knime.core.data.DataTableSpec) DataColumnSpecCreator(org.knime.core.data.DataColumnSpecCreator) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) SettingsModelString(org.knime.core.node.defaultnodesettings.SettingsModelString) RowInput(org.knime.core.node.streamable.RowInput) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) DataRow(org.knime.core.data.DataRow) UniqueNameGenerator(org.knime.core.util.UniqueNameGenerator) RowOutput(org.knime.core.node.streamable.RowOutput) ExecutionContext(org.knime.core.node.ExecutionContext) DataColumnSpec(org.knime.core.data.DataColumnSpec) DataCell(org.knime.core.data.DataCell) ReplacedColumnsDataRow(org.knime.base.data.replace.ReplacedColumnsDataRow) AppendedColumnRow(org.knime.core.data.append.AppendedColumnRow)

Example 4 with RowOutput

use of org.knime.core.node.streamable.RowOutput in project knime-core by knime.

the class SampleDataNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            DataTableSpec[] outSpecs = configure(new DataTableSpec[2]);
            DataTableSpec dataSpec = outSpecs[0];
            DataTableSpec clusterSpec = outSpecs[1];
            RowOutput dataOut = (RowOutput) outputs[0];
            RowOutput clusterOut = (RowOutput) outputs[1];
            run(dataSpec, dataOut, clusterSpec, clusterOut, exec);
        }
    };
}
Also used : DataTableSpec(org.knime.core.data.DataTableSpec) BufferedDataTableRowOutput(org.knime.core.node.streamable.BufferedDataTableRowOutput) RowOutput(org.knime.core.node.streamable.RowOutput) ExecutionContext(org.knime.core.node.ExecutionContext) StreamableOperator(org.knime.core.node.streamable.StreamableOperator)

Example 5 with RowOutput

use of org.knime.core.node.streamable.RowOutput in project knime-core by knime.

the class ColumnAppenderNodeModel method createStreamableOperator.

// ////////////// STREAMING FUNCTIONS ////////////////
/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            RowInput in1 = (RowInput) inputs[0];
            RowInput in2 = (RowInput) inputs[1];
            RowOutput out = (RowOutput) outputs[0];
            CustomRowIterator tableIt1 = new CustomRowIteratorImpl2(in1);
            CustomRowIterator tableIt2 = new CustomRowIteratorImpl2(in2);
            compute(tableIt1, tableIt2, in1.getDataTableSpec().getNumColumns() + in2.getDataTableSpec().getNumColumns(), row -> {
                out.push(row);
            }, exec, -1, -1);
            // poll all the remaining rows if there are any but don't do anything with them
            while (tableIt1.hasNext()) {
                tableIt1.next();
            }
            while (tableIt2.hasNext()) {
                tableIt2.next();
            }
            in1.close();
            in2.close();
            out.close();
        }
    };
}
Also used : RowOutput(org.knime.core.node.streamable.RowOutput) ExecutionContext(org.knime.core.node.ExecutionContext) StreamableOperator(org.knime.core.node.streamable.StreamableOperator) RowInput(org.knime.core.node.streamable.RowInput)

Aggregations

RowOutput (org.knime.core.node.streamable.RowOutput)28 ExecutionContext (org.knime.core.node.ExecutionContext)23 StreamableOperator (org.knime.core.node.streamable.StreamableOperator)23 RowInput (org.knime.core.node.streamable.RowInput)21 DataTableSpec (org.knime.core.data.DataTableSpec)16 DataRow (org.knime.core.data.DataRow)13 BufferedDataTableRowOutput (org.knime.core.node.streamable.BufferedDataTableRowOutput)13 DataTableRowInput (org.knime.core.node.streamable.DataTableRowInput)11 BufferedDataTable (org.knime.core.node.BufferedDataTable)9 ColumnRearranger (org.knime.core.data.container.ColumnRearranger)8 InvalidSettingsException (org.knime.core.node.InvalidSettingsException)8 DataCell (org.knime.core.data.DataCell)7 SettingsModelString (org.knime.core.node.defaultnodesettings.SettingsModelString)6 IOException (java.io.IOException)5 CanceledExecutionException (org.knime.core.node.CanceledExecutionException)5 StreamableOperatorInternals (org.knime.core.node.streamable.StreamableOperatorInternals)5 File (java.io.File)4 Arrays (java.util.Arrays)4 ReplacedColumnsDataRow (org.knime.base.data.replace.ReplacedColumnsDataRow)4 DataColumnSpec (org.knime.core.data.DataColumnSpec)4