Examples with BufferedDataContainer org.knime.core.node.BufferedDataContainer used on opensource projects

Example 51 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class RecursiveLoopEnd2NodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    if (!(this.getLoopStartNode() instanceof RecursiveLoopStart2NodeModel)) {
        throw new IllegalStateException("Loop End is not connected" + " to matching/corresponding Recursive Loop Start (2 ports) node.");
    }
    // in port 2: is fed back to loop start node
    BufferedDataContainer loopData = exec.createDataContainer(inData[resultingIn2].getDataTableSpec());
    ExecutionContext exec1 = exec.createSubExecutionContext(0.3);
    int count = 0;
    for (DataRow row : inData[resultingIn2]) {
        exec1.checkCanceled();
        exec1.setProgress(1.0 * count / loopData.size(), "Copy input table 2");
        loopData.addRowToTable(createNewRow(row, row.getKey()));
    }
    loopData.close();
    m_inData2 = loopData.getTable();
    return super.execute(inData, exec.createSubExecutionContext(0.7));
}

Example 52 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class AggregateOutputNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    // retrieve variables from the stack which the head of this
    // loop hopefully put there:
    int count;
    int maxCount;
    try {
        count = peekFlowVariableInt("currentIteration");
        maxCount = peekFlowVariableInt("maxIterations");
    } catch (NoSuchElementException e) {
        throw new Exception("No matching Loop Start node!", e);
    }
    if (count < 0 || count >= maxCount) {
        throw new Exception("Conflicting loop variables, count is " + count + " and max count is " + maxCount);
    }
    final BufferedDataTable in = inData[0];
    final DataTableSpec inSpec = in.getDataTableSpec();
    if (count == 0) {
        m_firstIterationSpec = in.getDataTableSpec();
        m_predictionTable = exec.createDataContainer(createPredictionSpec(in.getDataTableSpec()));
    } else if (m_predictionTable == null) {
        throw new Exception("Loop Head claims this is NOT the first iteration" + " but the tail believes it is?!");
    } else {
        if (!inSpec.equalStructure(m_firstIterationSpec)) {
            StringBuilder error = new StringBuilder("Input table's structure differs from reference " + "(first iteration) table: ");
            if (inSpec.getNumColumns() != m_firstIterationSpec.getNumColumns()) {
                error.append("different column counts ");
                error.append(inSpec.getNumColumns());
                error.append(" vs. ").append(m_firstIterationSpec.getNumColumns());
            } else {
                for (int i = 0; i < inSpec.getNumColumns(); i++) {
                    DataColumnSpec inCol = inSpec.getColumnSpec(i);
                    DataColumnSpec predCol = m_firstIterationSpec.getColumnSpec(i);
                    if (!inCol.equalStructure(predCol)) {
                        error.append("Column ").append(i).append(" [");
                        error.append(inCol).append("] vs. [");
                        error.append(predCol).append("]");
                    }
                }
            }
            throw new IllegalArgumentException(error.toString());
        }
    }
    final int rowCount = in.getRowCount();
    final int targetColIndex = in.getDataTableSpec().findColumnIndex(m_settings.targetColumn());
    final int predictColIndex = in.getDataTableSpec().findColumnIndex(m_settings.predictionColumn());
    final boolean numericMode = in.getDataTableSpec().getColumnSpec(predictColIndex).getType().isCompatible(DoubleValue.class);
    ExecutionMonitor subExec = exec.createSubProgress(count == maxCount - 1 ? 0.9 : 1);
    final DataCell foldNumber = new IntCell(m_foldStatistics.size());
    if (numericMode) {
        double errorSum = 0;
        int r = 0;
        for (DataRow row : in) {
            RowKey key = row.getKey();
            DoubleValue target = (DoubleValue) row.getCell(targetColIndex);
            DoubleValue predict = (DoubleValue) row.getCell(predictColIndex);
            double d = (target.getDoubleValue() - predict.getDoubleValue());
            errorSum += d * d;
            r++;
            if (m_settings.addFoldId()) {
                m_predictionTable.addRowToTable(new AppendedColumnRow(row.getKey(), row, foldNumber));
            } else {
                m_predictionTable.addRowToTable(row);
            }
            subExec.setProgress(r / (double) rowCount, "Calculating output " + r + "/" + rowCount + " (\"" + key + "\")");
            subExec.checkCanceled();
        }
        DataRow stats = new DefaultRow(new RowKey("fold " + m_foldStatistics.size()), new DoubleCell(errorSum), new DoubleCell(errorSum / rowCount), new IntCell(rowCount));
        m_foldStatistics.add(stats);
    } else {
        int incorrect = 0;
        int r = 0;
        for (DataRow row : in) {
            RowKey key = row.getKey();
            DataCell target = row.getCell(targetColIndex);
            DataCell predict = row.getCell(predictColIndex);
            if (!target.equals(predict)) {
                incorrect++;
            }
            r++;
            if (m_settings.addFoldId()) {
                m_predictionTable.addRowToTable(new AppendedColumnRow(row.getKey(), row, foldNumber));
            } else {
                m_predictionTable.addRowToTable(row);
            }
            subExec.setProgress(r / (double) rowCount, "Calculating output " + r + "/" + rowCount + " (\"" + key + "\")");
            subExec.checkCanceled();
        }
        DataRow stats = new DefaultRow(new RowKey("fold " + m_foldStatistics.size()), new DoubleCell(100.0 * incorrect / rowCount), new IntCell(rowCount), new IntCell(incorrect));
        m_foldStatistics.add(stats);
    }
    if (count < maxCount - 1) {
        continueLoop();
        return new BufferedDataTable[2];
    } else {
        BufferedDataContainer cont = exec.createDataContainer(numericMode ? NUMERIC_STATISTICS_SPEC : NOMINAL_STATISTICS_SPEC);
        for (DataRow row : m_foldStatistics) {
            cont.addRowToTable(row);
        }
        cont.close();
        m_predictionTable.close();
        return new BufferedDataTable[] { m_predictionTable.getTable(), cont.getTable() };
    }
}

Example 53 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class PolyRegLearnerNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
    BufferedDataTable inTable = (BufferedDataTable) inData[0];
    DataTableSpec inSpec = inTable.getDataTableSpec();
    final int colCount = inSpec.getNumColumns();
    String[] selectedCols = computeSelectedColumns(inSpec);
    Set<String> hash = new HashSet<String>(Arrays.asList(selectedCols));
    m_colSelected = new boolean[colCount];
    for (int i = 0; i < colCount; i++) {
        m_colSelected[i] = hash.contains(inTable.getDataTableSpec().getColumnSpec(i).getName());
    }
    final int rowCount = inTable.getRowCount();
    String[] temp = new String[m_columnNames.length + 1];
    System.arraycopy(m_columnNames, 0, temp, 0, m_columnNames.length);
    temp[temp.length - 1] = m_settings.getTargetColumn();
    FilterColumnTable filteredTable = new FilterColumnTable(inTable, temp);
    final DataArray rowContainer = new DefaultDataArray(filteredTable, 1, m_settings.getMaxRowsForView());
    // handle the optional PMML input
    PMMLPortObject inPMMLPort = m_pmmlInEnabled ? (PMMLPortObject) inData[1] : null;
    PortObjectSpec[] outputSpec = configure((inPMMLPort == null) ? new PortObjectSpec[] { inData[0].getSpec(), null } : new PortObjectSpec[] { inData[0].getSpec(), inPMMLPort.getSpec() });
    Learner learner = new Learner((PMMLPortObjectSpec) outputSpec[0], 0d, m_settings.getMissingValueHandling() == MissingValueHandling.fail, m_settings.getDegree());
    try {
        PolyRegContent polyRegContent = learner.perform(inTable, exec);
        m_betas = fillBeta(polyRegContent);
        m_meanValues = polyRegContent.getMeans();
        ColumnRearranger crea = new ColumnRearranger(inTable.getDataTableSpec());
        crea.append(getCellFactory(inTable.getDataTableSpec().findColumnIndex(m_settings.getTargetColumn())));
        PortObject[] bdt = new PortObject[] { createPMMLModel(inPMMLPort, inSpec), exec.createColumnRearrangeTable(inTable, crea, exec.createSilentSubExecutionContext(.2)), polyRegContent.createTablePortObject(exec.createSubExecutionContext(0.2)) };
        m_squaredError /= rowCount;
        if (polyRegContent.getWarningMessage() != null) {
            setWarningMessage(polyRegContent.getWarningMessage());
        }
        double[] stdErrors = PolyRegViewData.mapToArray(polyRegContent.getStandardErrors(), m_columnNames, m_settings.getDegree(), polyRegContent.getInterceptStdErr());
        double[] tValues = PolyRegViewData.mapToArray(polyRegContent.getTValues(), m_columnNames, m_settings.getDegree(), polyRegContent.getInterceptTValue());
        double[] pValues = PolyRegViewData.mapToArray(polyRegContent.getPValues(), m_columnNames, m_settings.getDegree(), polyRegContent.getInterceptPValue());
        m_viewData = new PolyRegViewData(m_meanValues, m_betas, stdErrors, tValues, pValues, m_squaredError, polyRegContent.getAdjustedRSquared(), m_columnNames, m_settings.getDegree(), m_settings.getTargetColumn(), rowContainer);
        return bdt;
    } catch (ModelSpecificationException e) {
        final String origWarning = getWarningMessage();
        final String warning = (origWarning != null && !origWarning.isEmpty()) ? (origWarning + "\n") : "" + e.getMessage();
        setWarningMessage(warning);
        final ExecutionContext subExec = exec.createSubExecutionContext(.1);
        final BufferedDataContainer empty = subExec.createDataContainer(STATS_SPEC);
        int rowIdx = 1;
        for (final String column : m_columnNames) {
            for (int d = 1; d <= m_settings.getDegree(); ++d) {
                empty.addRowToTable(new DefaultRow("Row" + rowIdx++, new StringCell(column), new IntCell(d), new DoubleCell(0.0d), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell()));
            }
        }
        empty.addRowToTable(new DefaultRow("Row" + rowIdx, new StringCell("Intercept"), new IntCell(0), new DoubleCell(0.0d), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell()));
        double[] nans = new double[m_columnNames.length * m_settings.getDegree() + 1];
        Arrays.fill(nans, Double.NaN);
        m_betas = new double[nans.length];
        // Mean only for the linear tags
        m_meanValues = new double[nans.length / m_settings.getDegree()];
        m_viewData = new PolyRegViewData(m_meanValues, m_betas, nans, nans, nans, m_squaredError, Double.NaN, m_columnNames, m_settings.getDegree(), m_settings.getTargetColumn(), rowContainer);
        empty.close();
        ColumnRearranger crea = new ColumnRearranger(inTable.getDataTableSpec());
        crea.append(getCellFactory(inTable.getDataTableSpec().findColumnIndex(m_settings.getTargetColumn())));
        BufferedDataTable rearrangerTable = exec.createColumnRearrangeTable(inTable, crea, exec.createSubProgress(0.6));
        PMMLPortObject model = createPMMLModel(inPMMLPort, inTable.getDataTableSpec());
        PortObject[] bdt = new PortObject[] { model, rearrangerTable, empty.getTable() };
        return bdt;
    }
}

Example 54 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class DecTreePredictorNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
public PortObject[] execute(final PortObject[] inPorts, final ExecutionContext exec) throws CanceledExecutionException, Exception {
    exec.setMessage("Decision Tree Predictor: Loading predictor...");
    PMMLPortObject port = (PMMLPortObject) inPorts[INMODELPORT];
    List<Node> models = port.getPMMLValue().getModels(PMMLModelType.TreeModel);
    if (models.isEmpty()) {
        String msg = "Decision Tree evaluation failed: " + "No tree model found.";
        LOGGER.error(msg);
        throw new RuntimeException(msg);
    }
    PMMLDecisionTreeTranslator trans = new PMMLDecisionTreeTranslator();
    port.initializeModelTranslator(trans);
    DecisionTree decTree = trans.getDecisionTree();
    decTree.resetColorInformation();
    BufferedDataTable inData = (BufferedDataTable) inPorts[INDATAPORT];
    // get column with color information
    String colorColumn = null;
    for (DataColumnSpec s : inData.getDataTableSpec()) {
        if (s.getColorHandler() != null) {
            colorColumn = s.getName();
            break;
        }
    }
    decTree.setColorColumn(colorColumn);
    exec.setMessage("Decision Tree Predictor: start execution.");
    PortObjectSpec[] inSpecs = new PortObjectSpec[] { inPorts[0].getSpec(), inPorts[1].getSpec() };
    DataTableSpec outSpec = createOutTableSpec(inSpecs);
    BufferedDataContainer outData = exec.createDataContainer(outSpec);
    long coveredPattern = 0;
    long nrPattern = 0;
    long rowCount = 0;
    final long numberRows = inData.size();
    exec.setMessage("Classifying...");
    List<String> predictionValues = getPredictionStrings((PMMLPortObjectSpec) inPorts[INMODELPORT].getSpec());
    for (DataRow thisRow : inData) {
        DataCell cl = null;
        LinkedHashMap<String, Double> classDistrib = null;
        try {
            Pair<DataCell, LinkedHashMap<DataCell, Double>> pair = decTree.getWinnerAndClasscounts(thisRow, inData.getDataTableSpec());
            cl = pair.getFirst();
            LinkedHashMap<DataCell, Double> classCounts = pair.getSecond();
            classDistrib = getDistribution(classCounts);
            if (coveredPattern < m_maxNumCoveredPattern.getIntValue()) {
                // remember this one for HiLite support
                decTree.addCoveredPattern(thisRow, inData.getDataTableSpec());
                coveredPattern++;
            } else {
                // too many patterns for HiLite - at least remember color
                decTree.addCoveredColor(thisRow, inData.getDataTableSpec());
            }
            nrPattern++;
        } catch (Exception e) {
            LOGGER.error("Decision Tree evaluation failed: " + e.getMessage());
            throw e;
        }
        if (cl == null) {
            LOGGER.error("Decision Tree evaluation failed: result empty");
            throw new Exception("Decision Tree evaluation failed.");
        }
        DataCell[] newCells = new DataCell[outSpec.getNumColumns()];
        int numInCells = thisRow.getNumCells();
        for (int i = 0; i < numInCells; i++) {
            newCells[i] = thisRow.getCell(i);
        }
        if (m_showDistribution.getBooleanValue()) {
            assert predictionValues.size() >= newCells.length - 1 - numInCells : "Could not determine the prediction values: " + newCells.length + "; " + numInCells + "; " + predictionValues;
            for (int i = numInCells; i < newCells.length - 1; i++) {
                String predClass = predictionValues.get(i - numInCells);
                if (classDistrib != null && classDistrib.get(predClass) != null) {
                    newCells[i] = new DoubleCell(classDistrib.get(predClass));
                } else {
                    newCells[i] = new DoubleCell(0.0);
                }
            }
        }
        newCells[newCells.length - 1] = cl;
        outData.addRowToTable(new DefaultRow(thisRow.getKey(), newCells));
        rowCount++;
        if (rowCount % 100 == 0) {
            exec.setProgress(rowCount / (double) numberRows, "Classifying... Row " + rowCount + " of " + numberRows);
        }
        exec.checkCanceled();
    }
    if (coveredPattern < nrPattern) {
        // let the user know that we did not store all available pattern
        // for HiLiting.
        this.setWarningMessage("Tree only stored first " + m_maxNumCoveredPattern.getIntValue() + " (of " + nrPattern + ") rows for HiLiting!");
    }
    outData.close();
    m_decTree = decTree;
    exec.setMessage("Decision Tree Predictor: end execution.");
    return new BufferedDataTable[] { outData.getTable() };
}

Example 55 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class PCANodeModel method createCovarianceTable.

/**
 * create data table from covariance matrix.
 *
 * @param exec
 *            execution context
 * @param m
 *            covariance matrix
 * @param inputColumnNames
 *            names of input columns the matrix was created from
 * @return table
 */
public static BufferedDataTable createCovarianceTable(final ExecutionContext exec, final double[][] m, final String[] inputColumnNames) {
    final BufferedDataContainer bdt = exec.createDataContainer(createCovarianceMatrixSpec(inputColumnNames));
    for (int i = 0; i < m.length; i++) {
        final DataCell[] cells = new DataCell[inputColumnNames.length];
        for (int j = 0; j < m[i].length; j++) {
            cells[j] = new DoubleCell(m[i][j]);
        }
        bdt.addRowToTable(new DefaultRow(inputColumnNames[i], cells));
    }
    bdt.close();
    final BufferedDataTable covarianceTable = bdt.getTable();
    return covarianceTable;
}