Examples with DataRow org.knime.core.data.DataRow used on opensource projects

Example 36 with DataRow

use of org.knime.core.data.DataRow in project knime-core by knime.

the class DateTimeToStringNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            final RowInput in = (RowInput) inputs[0];
            final RowOutput out = (RowOutput) outputs[0];
            final DataTableSpec inSpec = in.getDataTableSpec();
            final String[] includeList = m_colSelect.applyTo(inSpec).getIncludes();
            final int[] includeIndeces = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
            final boolean isReplace = m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE);
            DataRow row;
            while ((row = in.poll()) != null) {
                exec.checkCanceled();
                DataCell[] datacells = new DataCell[includeIndeces.length];
                for (int i = 0; i < includeIndeces.length; i++) {
                    if (isReplace) {
                        final DataColumnSpecCreator dataColumnSpecCreator = new DataColumnSpecCreator(includeList[i], StringCell.TYPE);
                        final TimeToStringCellFactory cellFac = new TimeToStringCellFactory(dataColumnSpecCreator.createSpec(), includeIndeces[i]);
                        datacells[i] = cellFac.getCell(row);
                    } else {
                        final DataColumnSpec dataColSpec = new UniqueNameGenerator(inSpec).newColumn(includeList[i] + m_suffix.getStringValue(), StringCell.TYPE);
                        final TimeToStringCellFactory cellFac = new TimeToStringCellFactory(dataColSpec, includeIndeces[i]);
                        datacells[i] = cellFac.getCell(row);
                    }
                }
                if (isReplace) {
                    out.push(new ReplacedColumnsDataRow(row, datacells, includeIndeces));
                } else {
                    out.push(new AppendedColumnRow(row, datacells));
                }
            }
            in.close();
            out.close();
        }
    };
}

Example 37 with DataRow

use of org.knime.core.data.DataRow in project knime-core by knime.

the class DateTimeBasedRowFilterNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    final BufferedDataTable dataTable = inData[0];
    final BufferedDataContainer container = exec.createDataContainer(dataTable.getDataTableSpec());
    // read input
    final int colIdx = dataTable.getDataTableSpec().findColumnIndex(m_colSelect.getStringValue());
    final ZonedDateTime executionStartTime = m_startAlwaysNow.getBooleanValue() ? ZonedDateTime.now() : null;
    final ZonedDateTime executionEndTime = m_endAlwaysNow.getBooleanValue() ? ZonedDateTime.now() : null;
    // filter rows
    for (final DataRow row : dataTable) {
        exec.checkCanceled();
        final DataCell cell = row.getCell(colIdx);
        if (!cell.isMissing()) {
            if (cell instanceof LocalDateValue && filterRowLocalDate(((LocalDateValue) cell).getLocalDate(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof LocalTimeValue && filterRowLocalTime(((LocalTimeValue) cell).getLocalTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof LocalDateTimeValue && filterRowLocalDateTime(((LocalDateTimeValue) cell).getLocalDateTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof ZonedDateTimeValue && filterRowZonedDateTime(((ZonedDateTimeValue) cell).getZonedDateTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            }
        }
    }
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 38 with DataRow

use of org.knime.core.data.DataRow in project knime-core by knime.

the class LoopStartWindowNodeModel method executeTemporalCentral.

/**
 * Computes the next window that shall be returned for time triggered events using central windowing.
 *
 * @param table that holds the data.
 * @param exec context of the execution.
 * @return Next window.
 */
private BufferedDataTable[] executeTemporalCentral(final BufferedDataTable table, final ExecutionContext exec) {
    BufferedDataContainer container = exec.createDataContainer(table.getSpec());
    int column = table.getDataTableSpec().findColumnIndex(m_timeColumnName);
    Duration startInterval = (Duration) getTemporalAmount(m_windowConfig.getTimeStepSize() + m_windowConfig.getTimeStepUnit().getUnitLetter());
    Duration windowDuration = (Duration) getTemporalAmount(m_windowConfig.getTimeWindowSize() + m_windowConfig.getTimeWindowUnit().getUnitLetter());
    /* To check if an overflow occurred concerning the current window */
    boolean overflow = false;
    // To check if an overflow occurred concerning next starting temporal.
    m_lastWindow = false;
    /* Compute end duration of window and beginning of next duration*/
    if (m_nextStartTemporal == null && m_rowIterator.hasNext()) {
        DataRow first = m_rowIterator.next();
        /* Check if column only consists of missing values. */
        while (first.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            first = m_rowIterator.next();
            printMissingWarning();
        }
        if (first.getCell(column).isMissing()) {
            m_logger.warn("Column '" + m_timeColumnName + "' only contains missing values.");
            container.close();
            return new BufferedDataTable[] { container.getTable() };
        }
        Temporal firstStart = getTemporal(first.getCell(column));
        m_prevTemporal = firstStart;
        /* Check if user specified start shall be used. */
        if (m_windowConfig.useSpecifiedStartTime()) {
            firstStart = m_timeConfig.getSelectedDateTime();
            Temporal firstEnd = firstStart.plus(windowDuration.dividedBy(2));
            /* Check for overflow. */
            if (compareTemporal(firstEnd, firstStart) <= 0) {
                overflow = true;
            }
            /* Move the window until the current end is greater or equal than the first row. */
            while (!overflow && compareTemporal(getTemporal(first.getCell(column)), firstEnd) > 0) {
                Temporal tempNextEnd = firstEnd.plus(startInterval);
                /* Check if next window yields an overflow. */
                if (compareTemporal(tempNextEnd, firstEnd) <= 0) {
                    // overflow = true;
                    break;
                }
                firstEnd = tempNextEnd;
            }
            /* Current start temporal, m_nextStartTemporal is used to re-use the skipTemporalWindow method. */
            m_nextStartTemporal = firstEnd.minus(windowDuration);
            m_windowEndTemporal = firstEnd;
            /* Check for underflow of the current start. */
            if (compareTemporal(m_nextStartTemporal, firstEnd) >= 0 && (!overflow || compareTemporal(m_nextStartTemporal, getMin(m_nextStartTemporal)) == 0)) {
                m_nextStartTemporal = getMin(m_nextStartTemporal);
                m_bufferedRows.add(first);
            } else {
                /* Skip window until we find one which contains at least one row. */
                skipTemporalWindow(first, column, startInterval, windowDuration);
                firstEnd = m_nextStartTemporal.plus(windowDuration);
                firstStart = firstEnd.minus(windowDuration.dividedBy(2));
                /* This fixes a bug (AP-8975) in the case where we only have to skip one window as the first applicable row lies within the
                     * next window. In this case the skipTemporalWindow simply returns without any changes. */
                Temporal temp = firstEnd.plus(startInterval);
                if (!m_bufferedRows.isEmpty() && compareTemporal(firstEnd, getTemporal(m_bufferedRows.getFirst().getCell(column))) < 0 && (compareTemporal(temp, temp.minus(windowDuration)) < 0 || compareTemporal(temp.minus(windowDuration), getTemporal(m_bufferedRows.getFirst().getCell(column))) < 0)) {
                    firstStart = firstStart.plus(startInterval);
                }
            }
            /* Check if we found a window which contains at least one row. */
            if (m_bufferedRows.isEmpty()) {
                container.close();
                m_logger.warn("No row lies within any of the possible windows.");
                return new BufferedDataTable[] { container.getTable() };
            }
        }
        m_windowEndTemporal = firstStart.plus(windowDuration.dividedBy(2));
        /* Might yield an underflow but is used to check if the current window is the last window.*/
        m_nextStartTemporal = firstStart.minus(windowDuration.dividedBy(2));
        if (compareTemporal(m_windowEndTemporal, firstStart) <= 0) {
            overflow = true;
            m_lastWindow = true;
        } else {
            Temporal tempNextEnd = m_windowEndTemporal.plus(startInterval);
            Temporal tempNextStart = tempNextEnd.minus(windowDuration);
            Temporal tempNextMid = tempNextEnd.minus(windowDuration.dividedBy(2));
            /* Check if the current window is the last window. */
            boolean nextEndOverflow = compareTemporal(tempNextEnd, m_windowEndTemporal) <= 0;
            boolean tempNextMidOverflow = nextEndOverflow && compareTemporal(tempNextMid, tempNextEnd) < 0;
            if (tempNextMidOverflow) {
                m_lastWindow = true;
            } else if (compareTemporal(tempNextEnd, m_windowEndTemporal) > 0 && compareTemporal(tempNextStart, tempNextEnd) >= 0) {
                /* Underflow occurred; set next start to minimum. */
                m_nextStartTemporal = getMin(tempNextStart);
            } else {
                m_nextStartTemporal = tempNextStart;
            }
        }
        if (!m_windowConfig.useSpecifiedStartTime()) {
            m_bufferedRows.add(first);
        }
    } else {
        m_prevTemporal = getTemporal(m_bufferedRows.getFirst().getCell(column));
        Temporal tempEnd = m_windowEndTemporal.plus(startInterval);
        /* Check for overflow of the window. */
        if (compareTemporal(tempEnd, m_windowEndTemporal) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = tempEnd;
            Temporal tempNextEnd = m_windowEndTemporal.plus(startInterval);
            Temporal tempNextStart = tempNextEnd.minus(windowDuration);
            Temporal currMid = m_windowEndTemporal.minus(windowDuration.dividedBy(2));
            Temporal tempNextMid = currMid.plus(startInterval);
            /* Check if the current window is the last window. */
            boolean nextEndOverflow = compareTemporal(tempNextEnd, m_windowEndTemporal) <= 0;
            boolean tempNextMidOverflow = nextEndOverflow && compareTemporal(tempNextMid, tempNextEnd) < 0;
            if (tempNextMidOverflow) {
                m_lastWindow = true;
            } else if (compareTemporal(tempNextEnd, m_windowEndTemporal) > 0 && compareTemporal(tempNextStart, tempNextEnd) >= 0) {
                /* Underflow occurred; set next start to minimum. */
                m_nextStartTemporal = getMin(tempNextStart);
            } else {
                m_nextStartTemporal = tempNextStart;
            }
        }
    }
    Iterator<DataRow> bufferedIterator = m_bufferedRows.iterator();
    boolean allBufferedRowsInWindow = true;
    /* Add buffered rows. */
    while (bufferedIterator.hasNext()) {
        DataRow row = bufferedIterator.next();
        Temporal temp = getTemporal(row.getCell(column));
        /* Checks if all buffered rows are in the specified window. */
        if (!overflow && compareTemporal(temp, m_windowEndTemporal) > 0) {
            allBufferedRowsInWindow = false;
            break;
        }
        container.addRowToTable(row);
        m_currRow++;
        if (overflow || m_lastWindow || compareTemporal(getTemporal(row.getCell(column)), m_nextStartTemporal) < 0) {
            bufferedIterator.remove();
        }
    }
    boolean lastEntryMissing = false;
    boolean addedNewToBuffer = false;
    /* Add newly read rows. */
    while (m_rowIterator.hasNext() && allBufferedRowsInWindow) {
        DataRow row = m_rowIterator.next();
        if (row.getCell(column).isMissing()) {
            printMissingWarning();
            lastEntryMissing = true;
            continue;
        }
        lastEntryMissing = false;
        Temporal currTemporal = getTemporal(row.getCell(column));
        /* Check if table is sorted in non-descending order according to temporal column. */
        if (compareTemporal(currTemporal, m_prevTemporal) < 0) {
            throw new IllegalStateException(m_orderException);
        }
        m_prevTemporal = currTemporal;
        /* Add rows for next window into the buffer. */
        if (!m_lastWindow && compareTemporal(currTemporal, m_nextStartTemporal) >= 0 && !overflow) {
            m_bufferedRows.add(row);
            addedNewToBuffer = true;
        }
        /* Add row to current output. */
        if (overflow || compareTemporal(currTemporal, m_windowEndTemporal) <= 0) {
            container.addRowToTable(row);
            /* The last entry has been in the current window, thus it is the last one. */
            if (!m_rowIterator.hasNext()) {
                m_lastWindow = true;
            }
            m_currRow++;
        } else {
            break;
        }
    }
    /* Checks if the last row we saw had a missing value. If this is the case the current window is the last window. */
    if (lastEntryMissing) {
        m_lastWindow = true;
    }
    /* Find next entry that lies in a following window. */
    DataRow row = null;
    /* Close iterator if last window has been filled. */
    if (m_lastWindow) {
        m_rowIterator.close();
    } else if (!allBufferedRowsInWindow) {
        /* Not all previously buffered rows are in the current window. */
        row = m_bufferedRows.remove();
    } else if (!m_rowIterator.hasNext() && !addedNewToBuffer) {
        /* We already returned the last row, but it would be in the next window. Nevertheless, terminate. */
        m_rowIterator.close();
        m_lastWindow = true;
    } else if (m_bufferedRows.size() == 0) {
        /* Buffer is empty, so get next row. */
        row = m_rowIterator.next();
        while (row.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            row = m_rowIterator.next();
            printMissingWarning();
        }
        if (row.getCell(column).isMissing()) {
            row = null;
            printMissingWarning();
        }
    } else if (!overflow && !m_bufferedRows.isEmpty()) {
        /* Checks if the next buffered row lies within the given window */
        if (compareTemporal(m_windowEndTemporal.plus(startInterval), m_windowEndTemporal) > 0) {
            Temporal temp = getTemporal(m_bufferedRows.getFirst().getCell(column));
            if (compareTemporal(temp, m_windowEndTemporal.plus(startInterval)) >= 0) {
                row = m_bufferedRows.removeFirst();
            }
        }
    }
    skipTemporalWindow(row, column, startInterval, windowDuration);
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 39 with DataRow

use of org.knime.core.data.DataRow in project knime-core by knime.

the class LoopStartWindowNodeModel method executeTemporalForward.

/**
 * Computes the next window that shall be returned for time triggered events using forward windowing.
 *
 * @param table that holds the data.
 * @param exec context of the execution.
 * @return Next window.
 */
private BufferedDataTable[] executeTemporalForward(final BufferedDataTable table, final ExecutionContext exec) {
    BufferedDataContainer container = exec.createDataContainer(table.getSpec());
    int column = table.getDataTableSpec().findColumnIndex(m_timeColumnName);
    TemporalAmount startInterval = getTemporalAmount(m_windowConfig.getTimeStepSize() + m_windowConfig.getTimeStepUnit().getUnitLetter());
    TemporalAmount windowDuration = getTemporalAmount(m_windowConfig.getTimeWindowSize() + m_windowConfig.getTimeWindowUnit().getUnitLetter());
    /* To check if an overflow occurred concerning the current window */
    boolean overflow = false;
    // To check if an overflow occurred concerning next starting temporal.
    m_lastWindow = false;
    /* Compute end duration of window and beginning of next duration*/
    if (m_nextStartTemporal == null && m_rowIterator.hasNext()) {
        DataRow first = m_rowIterator.next();
        /* Check if column only consists of missing values. */
        while (first.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            first = m_rowIterator.next();
            printMissingWarning();
        }
        if (first.getCell(column).isMissing()) {
            m_logger.warn("Column '" + m_timeColumnName + "' only contains missing values.");
            container.close();
            return new BufferedDataTable[] { container.getTable() };
        }
        Temporal firstStart = getTemporal(first.getCell(column));
        m_prevTemporal = getTemporal(first.getCell(column));
        /* Check if user specified start shall be used. */
        if (m_windowConfig.useSpecifiedStartTime()) {
            firstStart = m_timeConfig.getSelectedDateTime();
            /* Current start temporal, m_nextStartTemporal is used to re-use the skipTemporalWindow method. */
            m_nextStartTemporal = firstStart;
            m_windowEndTemporal = m_nextStartTemporal.plus(windowDuration);
            /* Check if overflow of current window occurs. If this is the case simply find first row of the given window. */
            if (compareTemporal(m_windowEndTemporal, m_nextStartTemporal) <= 0) {
                while (m_rowIterator.hasNext() && compareTemporal(getTemporal(first.getCell(column)), m_nextStartTemporal) < 0) {
                    first = m_rowIterator.next();
                }
                if (compareTemporal(getTemporal(first.getCell(column)), m_nextStartTemporal) >= 0) {
                    m_bufferedRows.addFirst(first);
                }
            } else {
                /* We may have to skip the temporal window and or the current rows to find a window containing at least one row. */
                skipTemporalWindow(first, column, startInterval, windowDuration);
                firstStart = m_nextStartTemporal;
                /* This fixes a bug (AP-8975) in the case where we only have to skip one window as the first applicable row lies within the
                     * next window. In this case the skipTemporalWindow simply returns without any changes. */
                if (!m_bufferedRows.isEmpty() && compareTemporal(getTemporal(m_bufferedRows.getFirst().getCell(column)), firstStart.plus(windowDuration)) > 0) {
                    firstStart = firstStart.plus(startInterval);
                }
            }
            if (m_bufferedRows.isEmpty()) {
                container.close();
                return new BufferedDataTable[] { container.getTable() };
            }
        }
        m_nextStartTemporal = firstStart.plus(startInterval);
        /* Check if the next starting temporal lies beyond the maximum temporal value. */
        if (compareTemporal(m_nextStartTemporal, firstStart) <= 0) {
            m_lastWindow = true;
        }
        /* Checks if window overflow occurs. */
        Temporal temp = firstStart.plus(windowDuration);
        if (compareTemporal(temp, firstStart) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = temp;
        }
        /* Add the first row if no user specified start is used. */
        if (!m_windowConfig.useSpecifiedStartTime()) {
            m_bufferedRows.add(first);
        }
    } else {
        m_prevTemporal = getTemporal(m_bufferedRows.getFirst().getCell(column));
        /* Checks if temporal overflow occurs. */
        Temporal temp = m_nextStartTemporal.plus(windowDuration);
        if (compareTemporal(temp, m_windowEndTemporal) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = temp;
            temp = m_nextStartTemporal.plus(startInterval);
            /* Check if the next starting temporal lies beyond the maximum temporal value. */
            if (compareTemporal(temp, m_nextStartTemporal) <= 0) {
                m_lastWindow = true;
            } else {
                m_nextStartTemporal = temp;
            }
        }
    }
    Iterator<DataRow> bufferedIterator = m_bufferedRows.iterator();
    boolean allBufferedRowsInWindow = true;
    /* Add buffered rows. */
    while (bufferedIterator.hasNext()) {
        DataRow row = bufferedIterator.next();
        Temporal temp = getTemporal(row.getCell(column));
        /* Checks if all buffered rows are in the specified window. */
        if (!overflow && compareTemporal(temp, m_windowEndTemporal) > 0) {
            allBufferedRowsInWindow = false;
            break;
        }
        container.addRowToTable(row);
        m_currRow++;
        if (overflow || m_lastWindow || compareTemporal(getTemporal(row.getCell(column)), m_nextStartTemporal) < 0) {
            bufferedIterator.remove();
        }
    }
    boolean lastEntryMissing = false;
    boolean addedNewToBuffer = false;
    /* Add newly read rows. */
    while (m_rowIterator.hasNext() && allBufferedRowsInWindow) {
        DataRow row = m_rowIterator.next();
        if (row.getCell(column).isMissing()) {
            printMissingWarning();
            lastEntryMissing = true;
            continue;
        }
        lastEntryMissing = false;
        Temporal currTemporal = getTemporal(row.getCell(column));
        /* Check if table is sorted in non-descending order according to temporal column. */
        if (compareTemporal(currTemporal, m_prevTemporal) < 0) {
            throw new IllegalStateException(m_orderException);
        }
        m_prevTemporal = currTemporal;
        /* Add rows for next window into the buffer. */
        if (!m_lastWindow && compareTemporal(currTemporal, m_nextStartTemporal) >= 0 && !overflow) {
            m_bufferedRows.add(row);
            addedNewToBuffer = true;
        }
        /* Add row to current output. */
        if (overflow || compareTemporal(currTemporal, m_windowEndTemporal) <= 0) {
            container.addRowToTable(row);
            /* The last entry has been in the current window, thus it is the last one. */
            if (!m_rowIterator.hasNext()) {
                m_lastWindow = true;
            }
            m_currRow++;
        } else {
            break;
        }
    }
    /* Checks if the last row we saw had a missing value. If this is the case the current window is the last window. */
    if (lastEntryMissing) {
        m_lastWindow = true;
    }
    /* Find next entry that lies in a following window. */
    DataRow row = null;
    /* Close iterator if last window has been filled. */
    if (m_lastWindow) {
        m_rowIterator.close();
    } else if (!allBufferedRowsInWindow) {
        /* Not all previously buffered rows are in the current window. */
        row = m_bufferedRows.remove();
    } else if (!m_rowIterator.hasNext() && !addedNewToBuffer) {
        /* We already returned the last row, but it would be in the next window. Nevertheless, terminate. */
        m_rowIterator.close();
        m_lastWindow = true;
    } else if (m_bufferedRows.size() == 0) {
        /* Buffer is empty, so get next row. */
        row = m_rowIterator.next();
        while (row.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            row = m_rowIterator.next();
            printMissingWarning();
        }
        if (row.getCell(column).isMissing()) {
            row = null;
            printMissingWarning();
        }
    } else if (!overflow && !m_bufferedRows.isEmpty()) {
        /* Checks if the next buffered row lies within the given window */
        if (compareTemporal(m_windowEndTemporal.plus(startInterval), m_windowEndTemporal) > 0) {
            Temporal temp = getTemporal(m_bufferedRows.getFirst().getCell(column));
            if (compareTemporal(temp, m_windowEndTemporal.plus(startInterval)) >= 0) {
                row = m_bufferedRows.removeFirst();
            }
        }
    }
    skipTemporalWindow(row, column, startInterval, windowDuration);
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 40 with DataRow

use of org.knime.core.data.DataRow in project knime-core by knime.

the class DateShiftConfigure method getTimeBasedValueCellFactory.

/**
 * @param spec the previous data table spec
 * @param col1Idx the column index of the numerical column to add
 * @param g the time field to modify (as defined by calendar constants)
 * @param conf the configuration object
 * @param time the configured time as Calendar
 * @return the cell factory
 */
public static SingleCellFactory getTimeBasedValueCellFactory(final DataTableSpec spec, final int col1Idx, final int g, final DateShiftConfigure conf, final Calendar time) {
    return new SingleCellFactory(createOutputColumnSpec(spec, conf.getNewColumnName().getStringValue())) {

        /**
         * Value for the new column is based on the values of two column of the row (first and second date column),
         * the selected granularity, and the fraction digits for rounding.
         *
         * @param row the current row
         * @return the difference between the two date values with the given granularity and rounding
         */
        @Override
        public DataCell getCell(final DataRow row) {
            DataCell cell1 = row.getCell(col1Idx);
            if ((cell1.isMissing())) {
                return DataType.getMissingCell();
            }
            Calendar c = (Calendar) time.clone();
            c.add(g, ((IntValue) cell1).getIntValue());
            return new DateAndTimeCell(c.getTimeInMillis(), conf.getHasDate().getBooleanValue(), conf.getHasTime().getBooleanValue(), conf.getHasMiliSeconds().getBooleanValue());
        }
    };
}