Examples with BufferedDataContainer org.knime.core.node.BufferedDataContainer used on opensource projects

Example 21 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class TwoSampleTTestNodeModel method getTestStatisticsTable.

/**
 * Get table with test statistics
 * @param result test statistic
 * @param exec the execution context
 * @return a combined table of the test statistic
 */
private BufferedDataTable getTestStatisticsTable(final TwoSampleTTestStatistics[] result, final ExecutionContext exec) {
    BufferedDataContainer cont = exec.createDataContainer(TwoSampleTTestStatistics.getTableSpec());
    int r = 0;
    for (int i = 0; i < result.length; i++) {
        for (List<DataCell> cells : result[i].getTTestCells()) {
            cont.addRowToTable(new DefaultRow(RowKey.createRowKey(r), cells));
            r++;
        }
    }
    cont.close();
    return cont.getTable();
}

Example 22 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class DateTimeBasedRowFilterNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    final BufferedDataTable dataTable = inData[0];
    final BufferedDataContainer container = exec.createDataContainer(dataTable.getDataTableSpec());
    // read input
    final int colIdx = dataTable.getDataTableSpec().findColumnIndex(m_colSelect.getStringValue());
    final ZonedDateTime executionStartTime = m_startAlwaysNow.getBooleanValue() ? ZonedDateTime.now() : null;
    final ZonedDateTime executionEndTime = m_endAlwaysNow.getBooleanValue() ? ZonedDateTime.now() : null;
    // filter rows
    for (final DataRow row : dataTable) {
        exec.checkCanceled();
        final DataCell cell = row.getCell(colIdx);
        if (!cell.isMissing()) {
            if (cell instanceof LocalDateValue && filterRowLocalDate(((LocalDateValue) cell).getLocalDate(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof LocalTimeValue && filterRowLocalTime(((LocalTimeValue) cell).getLocalTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof LocalDateTimeValue && filterRowLocalDateTime(((LocalDateTimeValue) cell).getLocalDateTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            } else if (cell instanceof ZonedDateTimeValue && filterRowZonedDateTime(((ZonedDateTimeValue) cell).getZonedDateTime(), executionStartTime, executionEndTime)) {
                container.addRowToTable(row);
            }
        }
    }
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 23 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class LoopStartWindowNodeModel method executeTemporalCentral.

/**
 * Computes the next window that shall be returned for time triggered events using central windowing.
 *
 * @param table that holds the data.
 * @param exec context of the execution.
 * @return Next window.
 */
private BufferedDataTable[] executeTemporalCentral(final BufferedDataTable table, final ExecutionContext exec) {
    BufferedDataContainer container = exec.createDataContainer(table.getSpec());
    int column = table.getDataTableSpec().findColumnIndex(m_timeColumnName);
    Duration startInterval = (Duration) getTemporalAmount(m_windowConfig.getTimeStepSize() + m_windowConfig.getTimeStepUnit().getUnitLetter());
    Duration windowDuration = (Duration) getTemporalAmount(m_windowConfig.getTimeWindowSize() + m_windowConfig.getTimeWindowUnit().getUnitLetter());
    /* To check if an overflow occurred concerning the current window */
    boolean overflow = false;
    // To check if an overflow occurred concerning next starting temporal.
    m_lastWindow = false;
    /* Compute end duration of window and beginning of next duration*/
    if (m_nextStartTemporal == null && m_rowIterator.hasNext()) {
        DataRow first = m_rowIterator.next();
        /* Check if column only consists of missing values. */
        while (first.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            first = m_rowIterator.next();
            printMissingWarning();
        }
        if (first.getCell(column).isMissing()) {
            m_logger.warn("Column '" + m_timeColumnName + "' only contains missing values.");
            container.close();
            return new BufferedDataTable[] { container.getTable() };
        }
        Temporal firstStart = getTemporal(first.getCell(column));
        m_prevTemporal = firstStart;
        /* Check if user specified start shall be used. */
        if (m_windowConfig.useSpecifiedStartTime()) {
            firstStart = m_timeConfig.getSelectedDateTime();
            Temporal firstEnd = firstStart.plus(windowDuration.dividedBy(2));
            /* Check for overflow. */
            if (compareTemporal(firstEnd, firstStart) <= 0) {
                overflow = true;
            }
            /* Move the window until the current end is greater or equal than the first row. */
            while (!overflow && compareTemporal(getTemporal(first.getCell(column)), firstEnd) > 0) {
                Temporal tempNextEnd = firstEnd.plus(startInterval);
                /* Check if next window yields an overflow. */
                if (compareTemporal(tempNextEnd, firstEnd) <= 0) {
                    // overflow = true;
                    break;
                }
                firstEnd = tempNextEnd;
            }
            /* Current start temporal, m_nextStartTemporal is used to re-use the skipTemporalWindow method. */
            m_nextStartTemporal = firstEnd.minus(windowDuration);
            m_windowEndTemporal = firstEnd;
            /* Check for underflow of the current start. */
            if (compareTemporal(m_nextStartTemporal, firstEnd) >= 0 && (!overflow || compareTemporal(m_nextStartTemporal, getMin(m_nextStartTemporal)) == 0)) {
                m_nextStartTemporal = getMin(m_nextStartTemporal);
                m_bufferedRows.add(first);
            } else {
                /* Skip window until we find one which contains at least one row. */
                skipTemporalWindow(first, column, startInterval, windowDuration);
                firstEnd = m_nextStartTemporal.plus(windowDuration);
                firstStart = firstEnd.minus(windowDuration.dividedBy(2));
                /* This fixes a bug (AP-8975) in the case where we only have to skip one window as the first applicable row lies within the
                     * next window. In this case the skipTemporalWindow simply returns without any changes. */
                Temporal temp = firstEnd.plus(startInterval);
                if (!m_bufferedRows.isEmpty() && compareTemporal(firstEnd, getTemporal(m_bufferedRows.getFirst().getCell(column))) < 0 && (compareTemporal(temp, temp.minus(windowDuration)) < 0 || compareTemporal(temp.minus(windowDuration), getTemporal(m_bufferedRows.getFirst().getCell(column))) < 0)) {
                    firstStart = firstStart.plus(startInterval);
                }
            }
            /* Check if we found a window which contains at least one row. */
            if (m_bufferedRows.isEmpty()) {
                container.close();
                m_logger.warn("No row lies within any of the possible windows.");
                return new BufferedDataTable[] { container.getTable() };
            }
        }
        m_windowEndTemporal = firstStart.plus(windowDuration.dividedBy(2));
        /* Might yield an underflow but is used to check if the current window is the last window.*/
        m_nextStartTemporal = firstStart.minus(windowDuration.dividedBy(2));
        if (compareTemporal(m_windowEndTemporal, firstStart) <= 0) {
            overflow = true;
            m_lastWindow = true;
        } else {
            Temporal tempNextEnd = m_windowEndTemporal.plus(startInterval);
            Temporal tempNextStart = tempNextEnd.minus(windowDuration);
            Temporal tempNextMid = tempNextEnd.minus(windowDuration.dividedBy(2));
            /* Check if the current window is the last window. */
            boolean nextEndOverflow = compareTemporal(tempNextEnd, m_windowEndTemporal) <= 0;
            boolean tempNextMidOverflow = nextEndOverflow && compareTemporal(tempNextMid, tempNextEnd) < 0;
            if (tempNextMidOverflow) {
                m_lastWindow = true;
            } else if (compareTemporal(tempNextEnd, m_windowEndTemporal) > 0 && compareTemporal(tempNextStart, tempNextEnd) >= 0) {
                /* Underflow occurred; set next start to minimum. */
                m_nextStartTemporal = getMin(tempNextStart);
            } else {
                m_nextStartTemporal = tempNextStart;
            }
        }
        if (!m_windowConfig.useSpecifiedStartTime()) {
            m_bufferedRows.add(first);
        }
    } else {
        m_prevTemporal = getTemporal(m_bufferedRows.getFirst().getCell(column));
        Temporal tempEnd = m_windowEndTemporal.plus(startInterval);
        /* Check for overflow of the window. */
        if (compareTemporal(tempEnd, m_windowEndTemporal) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = tempEnd;
            Temporal tempNextEnd = m_windowEndTemporal.plus(startInterval);
            Temporal tempNextStart = tempNextEnd.minus(windowDuration);
            Temporal currMid = m_windowEndTemporal.minus(windowDuration.dividedBy(2));
            Temporal tempNextMid = currMid.plus(startInterval);
            /* Check if the current window is the last window. */
            boolean nextEndOverflow = compareTemporal(tempNextEnd, m_windowEndTemporal) <= 0;
            boolean tempNextMidOverflow = nextEndOverflow && compareTemporal(tempNextMid, tempNextEnd) < 0;
            if (tempNextMidOverflow) {
                m_lastWindow = true;
            } else if (compareTemporal(tempNextEnd, m_windowEndTemporal) > 0 && compareTemporal(tempNextStart, tempNextEnd) >= 0) {
                /* Underflow occurred; set next start to minimum. */
                m_nextStartTemporal = getMin(tempNextStart);
            } else {
                m_nextStartTemporal = tempNextStart;
            }
        }
    }
    Iterator<DataRow> bufferedIterator = m_bufferedRows.iterator();
    boolean allBufferedRowsInWindow = true;
    /* Add buffered rows. */
    while (bufferedIterator.hasNext()) {
        DataRow row = bufferedIterator.next();
        Temporal temp = getTemporal(row.getCell(column));
        /* Checks if all buffered rows are in the specified window. */
        if (!overflow && compareTemporal(temp, m_windowEndTemporal) > 0) {
            allBufferedRowsInWindow = false;
            break;
        }
        container.addRowToTable(row);
        m_currRow++;
        if (overflow || m_lastWindow || compareTemporal(getTemporal(row.getCell(column)), m_nextStartTemporal) < 0) {
            bufferedIterator.remove();
        }
    }
    boolean lastEntryMissing = false;
    boolean addedNewToBuffer = false;
    /* Add newly read rows. */
    while (m_rowIterator.hasNext() && allBufferedRowsInWindow) {
        DataRow row = m_rowIterator.next();
        if (row.getCell(column).isMissing()) {
            printMissingWarning();
            lastEntryMissing = true;
            continue;
        }
        lastEntryMissing = false;
        Temporal currTemporal = getTemporal(row.getCell(column));
        /* Check if table is sorted in non-descending order according to temporal column. */
        if (compareTemporal(currTemporal, m_prevTemporal) < 0) {
            throw new IllegalStateException(m_orderException);
        }
        m_prevTemporal = currTemporal;
        /* Add rows for next window into the buffer. */
        if (!m_lastWindow && compareTemporal(currTemporal, m_nextStartTemporal) >= 0 && !overflow) {
            m_bufferedRows.add(row);
            addedNewToBuffer = true;
        }
        /* Add row to current output. */
        if (overflow || compareTemporal(currTemporal, m_windowEndTemporal) <= 0) {
            container.addRowToTable(row);
            /* The last entry has been in the current window, thus it is the last one. */
            if (!m_rowIterator.hasNext()) {
                m_lastWindow = true;
            }
            m_currRow++;
        } else {
            break;
        }
    }
    /* Checks if the last row we saw had a missing value. If this is the case the current window is the last window. */
    if (lastEntryMissing) {
        m_lastWindow = true;
    }
    /* Find next entry that lies in a following window. */
    DataRow row = null;
    /* Close iterator if last window has been filled. */
    if (m_lastWindow) {
        m_rowIterator.close();
    } else if (!allBufferedRowsInWindow) {
        /* Not all previously buffered rows are in the current window. */
        row = m_bufferedRows.remove();
    } else if (!m_rowIterator.hasNext() && !addedNewToBuffer) {
        /* We already returned the last row, but it would be in the next window. Nevertheless, terminate. */
        m_rowIterator.close();
        m_lastWindow = true;
    } else if (m_bufferedRows.size() == 0) {
        /* Buffer is empty, so get next row. */
        row = m_rowIterator.next();
        while (row.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            row = m_rowIterator.next();
            printMissingWarning();
        }
        if (row.getCell(column).isMissing()) {
            row = null;
            printMissingWarning();
        }
    } else if (!overflow && !m_bufferedRows.isEmpty()) {
        /* Checks if the next buffered row lies within the given window */
        if (compareTemporal(m_windowEndTemporal.plus(startInterval), m_windowEndTemporal) > 0) {
            Temporal temp = getTemporal(m_bufferedRows.getFirst().getCell(column));
            if (compareTemporal(temp, m_windowEndTemporal.plus(startInterval)) >= 0) {
                row = m_bufferedRows.removeFirst();
            }
        }
    }
    skipTemporalWindow(row, column, startInterval, windowDuration);
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 24 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class LoopStartWindowNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    BufferedDataTable table = inData[0];
    m_rowCount = table.size();
    if (m_currRow == 0) {
        m_rowIterator = table.iterator();
        m_bufferedRows = new LinkedList<>();
        m_nColumns = table.getSpec().getNumColumns();
        if (m_rowCount == 0) {
            BufferedDataContainer container = exec.createDataContainer(table.getSpec());
            container.close();
            return new BufferedDataTable[] { container.getTable() };
        }
    }
    switch(m_windowConfig.getWindowDefinition()) {
        case BACKWARD:
            if (m_windowConfig.getTrigger().equals(Trigger.ROW)) {
                return executeBackward(table, exec);
            }
            return executeTemporalBackward(table, exec);
        case CENTRAL:
            if (m_windowConfig.getTrigger().equals(Trigger.ROW)) {
                return executeCentral(table, exec);
            }
            return executeTemporalCentral(table, exec);
        case FORWARD:
            if (m_windowConfig.getTrigger().equals(Trigger.ROW)) {
                return executeForward(table, exec);
            }
            return executeTemporalForward(table, exec);
        default:
            return executeForward(table, exec);
    }
}

Example 25 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class LoopStartWindowNodeModel method executeTemporalForward.

/**
 * Computes the next window that shall be returned for time triggered events using forward windowing.
 *
 * @param table that holds the data.
 * @param exec context of the execution.
 * @return Next window.
 */
private BufferedDataTable[] executeTemporalForward(final BufferedDataTable table, final ExecutionContext exec) {
    BufferedDataContainer container = exec.createDataContainer(table.getSpec());
    int column = table.getDataTableSpec().findColumnIndex(m_timeColumnName);
    TemporalAmount startInterval = getTemporalAmount(m_windowConfig.getTimeStepSize() + m_windowConfig.getTimeStepUnit().getUnitLetter());
    TemporalAmount windowDuration = getTemporalAmount(m_windowConfig.getTimeWindowSize() + m_windowConfig.getTimeWindowUnit().getUnitLetter());
    /* To check if an overflow occurred concerning the current window */
    boolean overflow = false;
    // To check if an overflow occurred concerning next starting temporal.
    m_lastWindow = false;
    /* Compute end duration of window and beginning of next duration*/
    if (m_nextStartTemporal == null && m_rowIterator.hasNext()) {
        DataRow first = m_rowIterator.next();
        /* Check if column only consists of missing values. */
        while (first.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            first = m_rowIterator.next();
            printMissingWarning();
        }
        if (first.getCell(column).isMissing()) {
            m_logger.warn("Column '" + m_timeColumnName + "' only contains missing values.");
            container.close();
            return new BufferedDataTable[] { container.getTable() };
        }
        Temporal firstStart = getTemporal(first.getCell(column));
        m_prevTemporal = getTemporal(first.getCell(column));
        /* Check if user specified start shall be used. */
        if (m_windowConfig.useSpecifiedStartTime()) {
            firstStart = m_timeConfig.getSelectedDateTime();
            /* Current start temporal, m_nextStartTemporal is used to re-use the skipTemporalWindow method. */
            m_nextStartTemporal = firstStart;
            m_windowEndTemporal = m_nextStartTemporal.plus(windowDuration);
            /* Check if overflow of current window occurs. If this is the case simply find first row of the given window. */
            if (compareTemporal(m_windowEndTemporal, m_nextStartTemporal) <= 0) {
                while (m_rowIterator.hasNext() && compareTemporal(getTemporal(first.getCell(column)), m_nextStartTemporal) < 0) {
                    first = m_rowIterator.next();
                }
                if (compareTemporal(getTemporal(first.getCell(column)), m_nextStartTemporal) >= 0) {
                    m_bufferedRows.addFirst(first);
                }
            } else {
                /* We may have to skip the temporal window and or the current rows to find a window containing at least one row. */
                skipTemporalWindow(first, column, startInterval, windowDuration);
                firstStart = m_nextStartTemporal;
                /* This fixes a bug (AP-8975) in the case where we only have to skip one window as the first applicable row lies within the
                     * next window. In this case the skipTemporalWindow simply returns without any changes. */
                if (!m_bufferedRows.isEmpty() && compareTemporal(getTemporal(m_bufferedRows.getFirst().getCell(column)), firstStart.plus(windowDuration)) > 0) {
                    firstStart = firstStart.plus(startInterval);
                }
            }
            if (m_bufferedRows.isEmpty()) {
                container.close();
                return new BufferedDataTable[] { container.getTable() };
            }
        }
        m_nextStartTemporal = firstStart.plus(startInterval);
        /* Check if the next starting temporal lies beyond the maximum temporal value. */
        if (compareTemporal(m_nextStartTemporal, firstStart) <= 0) {
            m_lastWindow = true;
        }
        /* Checks if window overflow occurs. */
        Temporal temp = firstStart.plus(windowDuration);
        if (compareTemporal(temp, firstStart) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = temp;
        }
        /* Add the first row if no user specified start is used. */
        if (!m_windowConfig.useSpecifiedStartTime()) {
            m_bufferedRows.add(first);
        }
    } else {
        m_prevTemporal = getTemporal(m_bufferedRows.getFirst().getCell(column));
        /* Checks if temporal overflow occurs. */
        Temporal temp = m_nextStartTemporal.plus(windowDuration);
        if (compareTemporal(temp, m_windowEndTemporal) <= 0) {
            overflow = true;
        } else {
            m_windowEndTemporal = temp;
            temp = m_nextStartTemporal.plus(startInterval);
            /* Check if the next starting temporal lies beyond the maximum temporal value. */
            if (compareTemporal(temp, m_nextStartTemporal) <= 0) {
                m_lastWindow = true;
            } else {
                m_nextStartTemporal = temp;
            }
        }
    }
    Iterator<DataRow> bufferedIterator = m_bufferedRows.iterator();
    boolean allBufferedRowsInWindow = true;
    /* Add buffered rows. */
    while (bufferedIterator.hasNext()) {
        DataRow row = bufferedIterator.next();
        Temporal temp = getTemporal(row.getCell(column));
        /* Checks if all buffered rows are in the specified window. */
        if (!overflow && compareTemporal(temp, m_windowEndTemporal) > 0) {
            allBufferedRowsInWindow = false;
            break;
        }
        container.addRowToTable(row);
        m_currRow++;
        if (overflow || m_lastWindow || compareTemporal(getTemporal(row.getCell(column)), m_nextStartTemporal) < 0) {
            bufferedIterator.remove();
        }
    }
    boolean lastEntryMissing = false;
    boolean addedNewToBuffer = false;
    /* Add newly read rows. */
    while (m_rowIterator.hasNext() && allBufferedRowsInWindow) {
        DataRow row = m_rowIterator.next();
        if (row.getCell(column).isMissing()) {
            printMissingWarning();
            lastEntryMissing = true;
            continue;
        }
        lastEntryMissing = false;
        Temporal currTemporal = getTemporal(row.getCell(column));
        /* Check if table is sorted in non-descending order according to temporal column. */
        if (compareTemporal(currTemporal, m_prevTemporal) < 0) {
            throw new IllegalStateException(m_orderException);
        }
        m_prevTemporal = currTemporal;
        /* Add rows for next window into the buffer. */
        if (!m_lastWindow && compareTemporal(currTemporal, m_nextStartTemporal) >= 0 && !overflow) {
            m_bufferedRows.add(row);
            addedNewToBuffer = true;
        }
        /* Add row to current output. */
        if (overflow || compareTemporal(currTemporal, m_windowEndTemporal) <= 0) {
            container.addRowToTable(row);
            /* The last entry has been in the current window, thus it is the last one. */
            if (!m_rowIterator.hasNext()) {
                m_lastWindow = true;
            }
            m_currRow++;
        } else {
            break;
        }
    }
    /* Checks if the last row we saw had a missing value. If this is the case the current window is the last window. */
    if (lastEntryMissing) {
        m_lastWindow = true;
    }
    /* Find next entry that lies in a following window. */
    DataRow row = null;
    /* Close iterator if last window has been filled. */
    if (m_lastWindow) {
        m_rowIterator.close();
    } else if (!allBufferedRowsInWindow) {
        /* Not all previously buffered rows are in the current window. */
        row = m_bufferedRows.remove();
    } else if (!m_rowIterator.hasNext() && !addedNewToBuffer) {
        /* We already returned the last row, but it would be in the next window. Nevertheless, terminate. */
        m_rowIterator.close();
        m_lastWindow = true;
    } else if (m_bufferedRows.size() == 0) {
        /* Buffer is empty, so get next row. */
        row = m_rowIterator.next();
        while (row.getCell(column).isMissing() && m_rowIterator.hasNext()) {
            row = m_rowIterator.next();
            printMissingWarning();
        }
        if (row.getCell(column).isMissing()) {
            row = null;
            printMissingWarning();
        }
    } else if (!overflow && !m_bufferedRows.isEmpty()) {
        /* Checks if the next buffered row lies within the given window */
        if (compareTemporal(m_windowEndTemporal.plus(startInterval), m_windowEndTemporal) > 0) {
            Temporal temp = getTemporal(m_bufferedRows.getFirst().getCell(column));
            if (compareTemporal(temp, m_windowEndTemporal.plus(startInterval)) >= 0) {
                row = m_bufferedRows.removeFirst();
            }
        }
    }
    skipTemporalWindow(row, column, startInterval, windowDuration);
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}