Examples with AppendedColumnRow org.knime.base.data.append.column.AppendedColumnRow used on opensource projects

Example 1 with AppendedColumnRow

use of org.knime.base.data.append.column.AppendedColumnRow in project knime-core by knime.

the class AggregateOutputNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    // retrieve variables from the stack which the head of this
    // loop hopefully put there:
    int count;
    int maxCount;
    try {
        count = peekFlowVariableInt("currentIteration");
        maxCount = peekFlowVariableInt("maxIterations");
    } catch (NoSuchElementException e) {
        throw new Exception("No matching Loop Start node!", e);
    }
    if (count < 0 || count >= maxCount) {
        throw new Exception("Conflicting loop variables, count is " + count + " and max count is " + maxCount);
    }
    final BufferedDataTable in = inData[0];
    final DataTableSpec inSpec = in.getDataTableSpec();
    if (count == 0) {
        m_firstIterationSpec = in.getDataTableSpec();
        m_predictionTable = exec.createDataContainer(createPredictionSpec(in.getDataTableSpec()));
    } else if (m_predictionTable == null) {
        throw new Exception("Loop Head claims this is NOT the first iteration" + " but the tail believes it is?!");
    } else {
        if (!inSpec.equalStructure(m_firstIterationSpec)) {
            StringBuilder error = new StringBuilder("Input table's structure differs from reference " + "(first iteration) table: ");
            if (inSpec.getNumColumns() != m_firstIterationSpec.getNumColumns()) {
                error.append("different column counts ");
                error.append(inSpec.getNumColumns());
                error.append(" vs. ").append(m_firstIterationSpec.getNumColumns());
            } else {
                for (int i = 0; i < inSpec.getNumColumns(); i++) {
                    DataColumnSpec inCol = inSpec.getColumnSpec(i);
                    DataColumnSpec predCol = m_firstIterationSpec.getColumnSpec(i);
                    if (!inCol.equalStructure(predCol)) {
                        error.append("Column ").append(i).append(" [");
                        error.append(inCol).append("] vs. [");
                        error.append(predCol).append("]");
                    }
                }
            }
            throw new IllegalArgumentException(error.toString());
        }
    }
    final int rowCount = in.getRowCount();
    final int targetColIndex = in.getDataTableSpec().findColumnIndex(m_settings.targetColumn());
    final int predictColIndex = in.getDataTableSpec().findColumnIndex(m_settings.predictionColumn());
    final boolean numericMode = in.getDataTableSpec().getColumnSpec(predictColIndex).getType().isCompatible(DoubleValue.class);
    ExecutionMonitor subExec = exec.createSubProgress(count == maxCount - 1 ? 0.9 : 1);
    final DataCell foldNumber = new IntCell(m_foldStatistics.size());
    if (numericMode) {
        double errorSum = 0;
        int r = 0;
        for (DataRow row : in) {
            RowKey key = row.getKey();
            DoubleValue target = (DoubleValue) row.getCell(targetColIndex);
            DoubleValue predict = (DoubleValue) row.getCell(predictColIndex);
            double d = (target.getDoubleValue() - predict.getDoubleValue());
            errorSum += d * d;
            r++;
            if (m_settings.addFoldId()) {
                m_predictionTable.addRowToTable(new AppendedColumnRow(row.getKey(), row, foldNumber));
            } else {
                m_predictionTable.addRowToTable(row);
            }
            subExec.setProgress(r / (double) rowCount, "Calculating output " + r + "/" + rowCount + " (\"" + key + "\")");
            subExec.checkCanceled();
        }
        DataRow stats = new DefaultRow(new RowKey("fold " + m_foldStatistics.size()), new DoubleCell(errorSum), new DoubleCell(errorSum / rowCount), new IntCell(rowCount));
        m_foldStatistics.add(stats);
    } else {
        int incorrect = 0;
        int r = 0;
        for (DataRow row : in) {
            RowKey key = row.getKey();
            DataCell target = row.getCell(targetColIndex);
            DataCell predict = row.getCell(predictColIndex);
            if (!target.equals(predict)) {
                incorrect++;
            }
            r++;
            if (m_settings.addFoldId()) {
                m_predictionTable.addRowToTable(new AppendedColumnRow(row.getKey(), row, foldNumber));
            } else {
                m_predictionTable.addRowToTable(row);
            }
            subExec.setProgress(r / (double) rowCount, "Calculating output " + r + "/" + rowCount + " (\"" + key + "\")");
            subExec.checkCanceled();
        }
        DataRow stats = new DefaultRow(new RowKey("fold " + m_foldStatistics.size()), new DoubleCell(100.0 * incorrect / rowCount), new IntCell(rowCount), new IntCell(incorrect));
        m_foldStatistics.add(stats);
    }
    if (count < maxCount - 1) {
        continueLoop();
        return new BufferedDataTable[2];
    } else {
        BufferedDataContainer cont = exec.createDataContainer(numericMode ? NUMERIC_STATISTICS_SPEC : NOMINAL_STATISTICS_SPEC);
        for (DataRow row : m_foldStatistics) {
            cont.addRowToTable(row);
        }
        cont.close();
        m_predictionTable.close();
        return new BufferedDataTable[] { m_predictionTable.getTable(), cont.getTable() };
    }
}

Example 2 with AppendedColumnRow

use of org.knime.base.data.append.column.AppendedColumnRow in project knime-core by knime.

the class Unpivot2NodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    DataTableSpec inSpec = inData[0].getSpec();
    String[] retainedColumns = m_retainedColumns.applyTo(inSpec).getIncludes();
    String[] valueColumns = m_valueColumns.applyTo(inSpec).getIncludes();
    int[] valueColumnIndices = new int[valueColumns.length];
    for (int i = 0; i < valueColumnIndices.length; i++) {
        valueColumnIndices[i] = inSpec.findColumnIndex(valueColumns[i]);
    }
    int[] orderColumnIdx = new int[retainedColumns.length];
    for (int i = 0; i < orderColumnIdx.length; i++) {
        orderColumnIdx[i] = inSpec.findColumnIndex(retainedColumns[i]);
    }
    final double newRowCnt = inData[0].size() * valueColumns.length;
    final boolean enableHilite = m_enableHilite.getBooleanValue();
    LinkedHashMap<RowKey, Set<RowKey>> map = new LinkedHashMap<RowKey, Set<RowKey>>();
    DataTableSpec outSpec = createOutSpec(inSpec);
    BufferedDataContainer buf = exec.createDataContainer(outSpec);
    final boolean skipMissings = m_missingValues.getBooleanValue();
    for (DataRow row : inData[0]) {
        LinkedHashSet<RowKey> set = new LinkedHashSet<RowKey>();
        FilterColumnRow crow = new FilterColumnRow(row, orderColumnIdx);
        for (int i = 0; i < valueColumns.length; i++) {
            String colName = valueColumns[i];
            DataCell acell = row.getCell(valueColumnIndices[i]);
            if (acell.isMissing() && skipMissings) {
                // skip rows containing missing cells (in Value column(s))
                continue;
            }
            RowKey rowKey = RowKey.createRowKey(buf.size());
            if (enableHilite) {
                set.add(rowKey);
            }
            DefaultRow drow = new DefaultRow(rowKey, new StringCell(row.getKey().getString()), new StringCell(colName), acell);
            buf.addRowToTable(new AppendedColumnRow(rowKey, drow, crow));
            exec.checkCanceled();
            exec.setProgress(buf.size() / newRowCnt);
        }
        if (enableHilite) {
            map.put(crow.getKey(), set);
        }
    }
    buf.close();
    if (enableHilite) {
        m_trans.setMapper(new DefaultHiLiteMapper(map));
    } else {
        m_trans.setMapper(null);
    }
    return new BufferedDataTable[] { buf.getTable() };
}

Example 3 with AppendedColumnRow

use of org.knime.base.data.append.column.AppendedColumnRow in project knime-core by knime.

the class UnpivotNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    DataTableSpec inSpec = inData[0].getSpec();
    List<String> orderColumns = m_orderColumns.getIncludeList();
    List<String> valueColumns = m_valueColumns.getIncludeList();
    int[] orderColumnIdx = new int[orderColumns.size()];
    for (int i = 0; i < orderColumnIdx.length; i++) {
        orderColumnIdx[i] = inSpec.findColumnIndex(orderColumns.get(i));
    }
    final double newRowCnt = inData[0].getRowCount() * valueColumns.size();
    final boolean enableHilite = m_enableHilite.getBooleanValue();
    LinkedHashMap<RowKey, Set<RowKey>> map = new LinkedHashMap<RowKey, Set<RowKey>>();
    DataTableSpec outSpec = createOutSpec(inSpec);
    BufferedDataContainer buf = exec.createDataContainer(outSpec);
    for (DataRow row : inData[0]) {
        LinkedHashSet<RowKey> set = new LinkedHashSet<RowKey>();
        FilterColumnRow crow = new FilterColumnRow(row, orderColumnIdx);
        for (int i = 0; i < valueColumns.size(); i++) {
            String colName = valueColumns.get(i);
            DataCell acell = row.getCell(inSpec.findColumnIndex(colName));
            if (acell.isMissing() && m_missingValues.getBooleanValue()) {
                // skip rows containing missing cells (in Value column(s))
                continue;
            }
            RowKey rowKey = RowKey.createRowKey(buf.size());
            if (enableHilite) {
                set.add(rowKey);
            }
            DefaultRow drow = new DefaultRow(rowKey, new StringCell(row.getKey().getString()), new StringCell(colName), acell);
            buf.addRowToTable(new AppendedColumnRow(rowKey, drow, crow));
            exec.checkCanceled();
            exec.setProgress(buf.size() / newRowCnt);
        }
        if (enableHilite) {
            map.put(crow.getKey(), set);
        }
    }
    buf.close();
    if (enableHilite) {
        m_trans.setMapper(new DefaultHiLiteMapper(map));
    } else {
        m_trans.setMapper(null);
    }
    return new BufferedDataTable[] { buf.getTable() };
}

Example 4 with AppendedColumnRow

use of org.knime.base.data.append.column.AppendedColumnRow in project knime-core by knime.

the class AppendedRowsIterator method initNextRow.

/**
 * Get next row internally.
 */
private void initNextRow() {
    // reached end of table's iterator - take next
    if (!m_curIterator.hasNext()) {
        do {
            if (m_curItIndex < m_iteratorSuppliers.length - 1) {
                initNextTable();
            } else {
                // final end
                m_nextRow = null;
                // reached end of this table
                return;
            }
        } while (!m_curIterator.hasNext());
    }
    // row from table
    DataRow baseRow = m_curIterator.next();
    m_curRowIndex++;
    boolean keyHasChanged = false;
    RowKey origKey = baseRow.getKey();
    RowKey key = origKey;
    while (m_duplicateMap.containsKey(key)) {
        if (m_exec != null) {
            try {
                m_exec.checkCanceled();
            } catch (CanceledExecutionException cee) {
                throw new RuntimeCanceledExecutionException(cee);
            }
        }
        switch(m_duplPolicy) {
            case Fail:
                assert false : "Duplicate checking is done in the BDT";
                throw new RuntimeException("Duplicate key \"" + key + "\"");
            case Skip:
                if (!m_hasPrintedError) {
                    LOGGER.warn("Table contains duplicate entry \"" + key.toString() + "\", skipping this row. " + "Suppress further warnings.");
                    m_hasPrintedError = true;
                }
                if (!m_curIterator.hasNext()) {
                    // end of one table reached
                    // note, this causes one more call on the stack
                    // (but who wants to concatenate 60000 tables...)
                    initNextRow();
                    return;
                }
                if (m_exec != null) {
                    m_nrRowsSkipped++;
                    String message = "Skipping row " + m_curRowIndex + " (\"" + key.toString() + "\")";
                    if (m_totalRowCount > 0L) {
                        m_exec.setProgress(m_curRowIndex / (double) m_totalRowCount, message);
                    } else {
                        m_exec.setMessage(message);
                    }
                }
                // row from table
                baseRow = m_curIterator.next();
                m_curRowIndex++;
                // stays false! rows have been skipped.
                keyHasChanged = false;
                origKey = baseRow.getKey();
                key = origKey;
                break;
            case AppendSuffix:
                // first time we come here
                if (!keyHasChanged && m_exec != null) {
                    String message = "Unifying row " + m_curRowIndex + " (\"" + key.toString() + "\")";
                    if (m_totalRowCount > 0L) {
                        m_exec.setProgress(m_curRowIndex / (double) m_totalRowCount, message);
                    } else {
                        m_exec.setMessage(message);
                    }
                }
                keyHasChanged = true;
                String newId = key.toString() + m_suffix;
                key = new RowKey(newId);
                // to do duplicate handling.
                break;
            default:
                throw new RuntimeException("Unknown policy: " + m_duplPolicy);
        }
    }
    switch(m_duplPolicy) {
        case Fail:
            // to do a efficient duplicate checking
            break;
        default:
            m_duplicateMap.put(key, origKey);
    }
    if (m_exec != null) {
        try {
            m_exec.checkCanceled();
        } catch (CanceledExecutionException cee) {
            throw new RuntimeCanceledExecutionException(cee);
        }
        String message = "Adding row " + m_curRowIndex + " (\"" + key.toString() + "\"" + (keyHasChanged ? " uniquified)" : ")");
        if (m_totalRowCount > 0L) {
            m_exec.setProgress(m_curRowIndex / (double) m_totalRowCount, message);
        } else {
            m_exec.setMessage(message);
        }
    }
    DataRow nextRow;
    if (m_curMissingCells != null) {
        // no missing cells implies the base row is complete
        assert (m_curMissingCells.length + baseRow.getNumCells() == m_spec.getNumColumns());
        // row enlarged by "missing" columns
        DataRow filledBaseRow = new AppendedColumnRow(baseRow, m_curMissingCells);
        nextRow = new ResortedCellsRow(filledBaseRow, m_curMapping);
    } else {
        nextRow = baseRow;
    }
    if (keyHasChanged) {
        DataCell[] cells = new DataCell[nextRow.getNumCells()];
        for (int i = 0; i < cells.length; i++) {
            cells[i] = nextRow.getCell(i);
        }
        m_nextRow = new DefaultRow(key, cells);
    } else {
        m_nextRow = nextRow;
    }
}

Example 5 with AppendedColumnRow

use of org.knime.base.data.append.column.AppendedColumnRow in project knime-core by knime.

the class LoopEndConditionNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    int count = peekFlowVariableInt("currentIteration");
    exec.setMessage("Iteration " + count);
    DataTableSpec spec1 = createSpec1(inData[0].getDataTableSpec());
    if (m_collectContainer == null) {
        assert m_variableContainer == null;
        m_startTime = System.currentTimeMillis();
        // first time we are getting to this: open container
        m_collectContainer = exec.createDataContainer(spec1);
        m_variableContainer = exec.createDataContainer(createSpec2());
    } else if (!spec1.equalStructure(m_collectContainer.getTableSpec())) {
        DataTableSpec predSpec = m_collectContainer.getTableSpec();
        StringBuilder error = new StringBuilder("Input table's structure differs from reference " + "(first iteration) table: ");
        if (spec1.getNumColumns() != predSpec.getNumColumns()) {
            error.append("different column counts ");
            error.append(spec1.getNumColumns());
            error.append(" vs. ").append(predSpec.getNumColumns());
        } else {
            for (int i = 0; i < spec1.getNumColumns(); i++) {
                DataColumnSpec inCol = spec1.getColumnSpec(i);
                DataColumnSpec predCol = predSpec.getColumnSpec(i);
                if (!inCol.equalStructure(predCol)) {
                    error.append("Column ").append(i).append(" [");
                    error.append(inCol).append("] vs. [");
                    error.append(predCol).append("]");
                }
            }
        }
        throw new IllegalArgumentException(error.toString());
    }
    RowKey rk = new RowKey("Iteration " + count);
    if (m_settings.variableType() == Type.DOUBLE) {
        m_variableContainer.addRowToTable(new DefaultRow(rk, new DoubleCell(peekFlowVariableDouble(m_settings.variableName()))));
    } else if (m_settings.variableType() == Type.INTEGER) {
        m_variableContainer.addRowToTable(new DefaultRow(rk, new IntCell(peekFlowVariableInt(m_settings.variableName()))));
    } else {
        m_variableContainer.addRowToTable(new DefaultRow(rk, new StringCell(peekFlowVariableString(m_settings.variableName()))));
    }
    LoopStartNode lsn = getLoopStartNode();
    boolean stop = checkCondition() || ((lsn instanceof LoopStartNodeTerminator) && ((LoopStartNodeTerminator) lsn).terminateLoop());
    if ((m_settings.addLastRows() && !m_settings.addLastRowsOnly()) || ((stop == m_settings.addLastRows()) && (stop == m_settings.addLastRowsOnly()))) {
        exec.setMessage("Collecting rows from current iteration");
        int k = 0;
        final double max = inData[0].size();
        IntCell currIterCell = new IntCell(count);
        for (DataRow row : inData[0]) {
            exec.checkCanceled();
            if (k++ % 10 == 0) {
                exec.setProgress(k / max);
            }
            DataRow newRow = new DefaultRow(new RowKey(row.getKey() + "#" + count), row);
            if (m_settings.addIterationColumn()) {
                newRow = new AppendedColumnRow(newRow, currIterCell);
            }
            m_collectContainer.addRowToTable(newRow);
        }
    }
    if (stop) {
        m_collectContainer.close();
        m_variableContainer.close();
        BufferedDataTable out1 = m_collectContainer.getTable();
        BufferedDataTable out2 = m_variableContainer.getTable();
        LOGGER.debug("Total loop execution time: " + (System.currentTimeMillis() - m_startTime) + "ms");
        m_startTime = 0;
        return new BufferedDataTable[] { out1, out2 };
    } else {
        continueLoop();
        return new BufferedDataTable[2];
    }
}