Examples with BufferedDataContainer org.knime.core.node.BufferedDataContainer used on opensource projects

Example 46 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class SubgroupMinerModel2 method createAssociationRulesOutput.

private BufferedDataTable createAssociationRulesOutput(final DataTableSpec inputSpec, final ExecutionContext exec, final AprioriAlgorithm apriori, final List<DataCell> nameMapping) {
    DataTableSpec outSpec = createAssociationRulesSpec(inputSpec);
    BufferedDataContainer ruleRows = exec.createDataContainer(outSpec);
    assert nameMapping != null;
    List<AssociationRule> associationRules = apriori.getAssociationRules(m_confidence.getDoubleValue());
    // for every association rule
    int rowKeyCounter = 0;
    for (AssociationRule r : associationRules) {
        // get the support
        double support = r.getSupport();
        // get the confidence
        double confidence = r.getConfidence();
        // get lift
        double lift = r.getLift();
        // get the antecedence (which is one item) -> cell
        FrequentItemSet antecedent = r.getAntecedent();
        // get the consequence
        FrequentItemSet consequent = r.getConsequent();
        DataCell[] allCells = new DataCell[6];
        allCells[0] = new DoubleCell(support);
        allCells[1] = new DoubleCell(confidence);
        allCells[2] = new DoubleCell(lift);
        // consequent is always only one item -> access with get(0) ok
        if (nameMapping.size() > consequent.getItems().get(0)) {
            allCells[3] = nameMapping.get(consequent.getItems().get(0));
        } else {
            allCells[3] = new StringCell("Item" + consequent.getItems().get(0));
        }
        allCells[4] = new StringCell("<---");
        Set<DataCell> allcells = new HashSet<DataCell>();
        for (int i = 0; i < antecedent.getItems().size() && i < m_maxItemSetLength.getIntValue() + 5; i++) {
            if (nameMapping.size() > antecedent.getItems().get(i)) {
                allcells.add(nameMapping.get(antecedent.getItems().get(i)));
            } else {
                allcells.add(new StringCell("Item" + antecedent.getItems().get(i)));
            }
        }
        allCells[5] = CollectionCellFactory.createSetCell(allcells);
        if (antecedent.getItems().size() > 0) {
            DataRow row = new DefaultRow("rule" + (rowKeyCounter++), allCells);
            ruleRows.addRowToTable(row);
        }
    }
    ruleRows.close();
    return ruleRows.getTable();
}

Example 47 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class AbstractParallelNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected final BufferedDataTable[] execute(final BufferedDataTable[] data, final ExecutionContext exec) throws Exception {
    final DataTableSpec[] outSpecs = prepareExecute(data);
    final List<Future<BufferedDataContainer[]>> futures = new ArrayList<>();
    final BufferedDataTable[] additionalTables = new BufferedDataTable[Math.max(0, data.length - 1)];
    System.arraycopy(data, 1, additionalTables, 0, additionalTables.length);
    // do some consistency checks to bail out as early as possible
    if (outSpecs == null) {
        throw new NullPointerException("Implementation Error: The " + "array of generated output table specs can't be null.");
    }
    if (outSpecs.length != getNrOutPorts()) {
        throw new IllegalStateException("Implementation Error: Number of" + " provided DataTableSpecs doesn't match number of output" + " ports");
    }
    for (DataTableSpec outSpec : outSpecs) {
        if (outSpec == null) {
            throw new IllegalStateException("Implementation Error: The" + " generated output DataTableSpec is null.");
        }
    }
    final double max = data[0].size();
    final Callable<Void> submitter = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            final RowIterator it = data[0].iterator();
            BufferedDataContainer container = null;
            int count = 0, chunks = 0;
            while (true) {
                if ((count++ % m_chunkSize == 0) || !it.hasNext()) {
                    exec.checkCanceled();
                    if (container != null) {
                        container.close();
                        final BufferedDataContainer temp = container;
                        chunks++;
                        final int temp2 = chunks;
                        futures.add(m_workers.submit(new Callable<BufferedDataContainer[]>() {

                            @Override
                            public BufferedDataContainer[] call() throws Exception {
                                ExecutionMonitor subProg = exec.createSilentSubProgress((m_chunkSize > max) ? 1 : m_chunkSize / max);
                                exec.setMessage("Processing chunk " + temp2);
                                BufferedDataContainer[] result = new BufferedDataContainer[outSpecs.length];
                                for (int i = 0; i < outSpecs.length; i++) {
                                    result[i] = exec.createDataContainer(outSpecs[i], true, 0);
                                }
                                executeByChunk(temp.getTable(), additionalTables, result, subProg);
                                for (DataContainer c : result) {
                                    c.close();
                                }
                                exec.setProgress(temp2 * m_chunkSize / max);
                                return result;
                            }
                        }));
                    }
                    if (!it.hasNext()) {
                        break;
                    }
                    container = exec.createDataContainer(data[0].getDataTableSpec());
                }
                container.addRowToTable(it.next());
            }
            return null;
        }
    };
    try {
        m_workers.runInvisible(submitter);
    } catch (IllegalThreadStateException ex) {
        // this node has not been started by a thread from a thread pool.
        // This is odd, but may happen
        submitter.call();
    }
    final BufferedDataTable[][] tempTables = new BufferedDataTable[outSpecs.length][futures.size()];
    int k = 0;
    for (Future<BufferedDataContainer[]> results : futures) {
        try {
            exec.checkCanceled();
        } catch (CanceledExecutionException ex) {
            for (Future<BufferedDataContainer[]> cancel : futures) {
                cancel.cancel(true);
            }
            throw ex;
        }
        final BufferedDataContainer[] temp = results.get();
        if ((temp == null) || (temp.length != getNrOutPorts())) {
            throw new IllegalStateException("Invalid result. Execution " + " failed, reason: data is null or number " + "of outputs wrong.");
        }
        for (int i = 0; i < temp.length; i++) {
            tempTables[i][k] = temp[i].getTable();
        }
        k++;
    }
    final BufferedDataTable[] resultTables = new BufferedDataTable[outSpecs.length];
    for (int i = 0; i < resultTables.length; i++) {
        resultTables[i] = exec.createConcatenateTable(exec, tempTables[i]);
    }
    return resultTables;
}

Example 48 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class ThreadedTableBuilderNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected final BufferedDataTable[] execute(final BufferedDataTable[] data, final ExecutionContext exec) throws Exception {
    final DataTableSpec[] outSpecs = prepareExecute(data);
    // do some consistency checks to bail out as early as possible
    if (outSpecs == null) {
        throw new NullPointerException("Implementation Error: The " + "array of generated output table specs can't be null.");
    }
    if (outSpecs.length != getNrOutPorts()) {
        throw new IllegalStateException("Implementation Error: Number of" + " provided DataTableSpecs doesn't match number of output" + " ports");
    }
    for (DataTableSpec outSpec : outSpecs) {
        if (outSpec == null) {
            throw new IllegalStateException("Implementation Error: The" + " generated output DataTableSpec is null.");
        }
    }
    final List<Future<BufferedDataContainer[]>> futures = new ArrayList<Future<BufferedDataContainer[]>>();
    final BufferedDataTable[] additionalTables = new BufferedDataTable[Math.max(0, data.length - 1)];
    System.arraycopy(data, 1, additionalTables, 0, additionalTables.length);
    final Callable<?> submitter = new Submitter(data, futures, outSpecs, exec);
    try {
        m_workers.runInvisible(submitter);
    } catch (IllegalThreadStateException ex) {
        // this node has not been started by a thread from a thread pool.
        // This is odd, but may happen
        submitter.call();
    }
    final BufferedDataTable[][] tempTables = new BufferedDataTable[outSpecs.length][futures.size()];
    int k = 0;
    for (Future<BufferedDataContainer[]> results : futures) {
        try {
            exec.checkCanceled();
        } catch (CanceledExecutionException ex) {
            for (Future<BufferedDataContainer[]> cancel : futures) {
                cancel.cancel(true);
            }
            throw ex;
        }
        final BufferedDataContainer[] temp = results.get();
        if ((temp == null) || (temp.length != getNrOutPorts())) {
            throw new IllegalStateException("Invalid result. Execution " + "failed, reason: data is null or number " + "of outputs wrong.");
        }
        for (int i = 0; i < temp.length; i++) {
            tempTables[i][k] = temp[i].getTable();
        }
        k++;
    }
    final BufferedDataTable[] resultTables = new BufferedDataTable[outSpecs.length];
    for (int i = 0; i < resultTables.length; i++) {
        resultTables[i] = exec.createConcatenateTable(exec, tempTables[i]);
    }
    return resultTables;
}

Example 49 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class AddEmptyRowsNodeModel method createNewRowsTable.

private BufferedDataTable createNewRowsTable(final DataTableSpec inSpec, final long rowCount, final ExecutionContext subExec) throws CanceledExecutionException {
    DataCell[] cells = new DataCell[inSpec.getNumColumns()];
    for (int c = 0; c < cells.length; c++) {
        DataType type = inSpec.getColumnSpec(c).getType();
        if (type.isASuperTypeOf(DoubleCell.TYPE)) {
            if (m_config.isUseMissingDouble()) {
                cells[c] = DataType.getMissingCell();
            } else {
                cells[c] = new DoubleCell(m_config.getFillValueDouble());
            }
        } else if (type.isASuperTypeOf(IntCell.TYPE)) {
            if (m_config.isUseMissingInt()) {
                cells[c] = DataType.getMissingCell();
            } else {
                cells[c] = new IntCell(m_config.getFillValueInt());
            }
        } else if (type.isASuperTypeOf(StringCell.TYPE)) {
            if (m_config.isUseMissingString()) {
                cells[c] = DataType.getMissingCell();
            } else {
                cells[c] = new StringCell(m_config.getFillValueString());
            }
        } else {
            cells[c] = DataType.getMissingCell();
        }
    }
    BufferedDataContainer cont = subExec.createDataContainer(inSpec);
    for (long i = 0; i < rowCount; i++) {
        RowKey key = new RowKey(m_config.getNewRowKeyPrefix() + i);
        subExec.setProgress(i / (double) rowCount, "Creating row \"" + key + "\", " + i + "/" + rowCount);
        subExec.checkCanceled();
        cont.addRowToTable(new DefaultRow(key, cells));
    }
    cont.close();
    return cont.getTable();
}

Example 50 with BufferedDataContainer

use of org.knime.core.node.BufferedDataContainer in project knime-core by knime.

the class GroupLoopStartNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    // /////////////////////////
    // 
    // / DATA TABLES (SORTING)
    // 
    // /////////////////////////
    BufferedDataTable table = inData[0];
    DataTableSpec spec = table.getDataTableSpec();
    if (table.size() <= 0) {
        m_endLoop = true;
    }
    // parameters
    m_includedColIndices = getIncludedColIndices(table.getDataTableSpec());
    boolean checkDuplicates = m_sortedInputTableModel.getBooleanValue();
    // remember table and sort table if necessary
    if (m_iteration == 0) {
        assert getLoopEndNode() == null : "1st iteration but end node set";
        m_table = table;
        m_spec = m_table.getDataTableSpec();
        // sort if not already sorted
        if (!m_sortedInputTableModel.getBooleanValue()) {
            // asc
            final String[] includes = m_filterGroupColModel.applyTo(spec).getIncludes();
            boolean[] sortAsc = new boolean[includes.length];
            Arrays.fill(sortAsc, true);
            BufferedDataTableSorter tableSorter = new BufferedDataTableSorter(table, Arrays.asList(includes), sortAsc, false);
            m_sortedTable = tableSorter.sort(exec);
        } else {
            // no sort necessary
            m_sortedTable = table;
        }
        m_iterator = m_sortedTable.iterator();
    } else {
        assert getLoopEndNode() != null : "No end node set";
        assert table == m_table : "Input tables differ between iterations";
    }
    // /////////////////////////
    // 
    // / INIT
    // 
    // /////////////////////////
    BufferedDataContainer cont = exec.createDataContainer(table.getSpec());
    // create new duplicate checker if null
    if (m_duplicateChecker == null) {
        m_duplicateChecker = new DuplicateChecker();
    }
    // initialize grouping states if null
    if (m_currentGroupingState == null) {
        m_currentGroupingState = new GroupingState("", false, null);
    }
    m_lastGroupingState = m_currentGroupingState;
    // add now to new group
    if (m_lastRow != null) {
        cont.addRowToTable(m_lastRow);
    }
    // if the final row has been reached and added set end loop flag
    if (m_isFinalRow) {
        m_endLoop = true;
    }
    // walk trough input table and group data
    // as long as new row fits into the current group or there are no more
    // rows left.
    boolean groupEnd = false;
    while (!groupEnd && m_iterator.hasNext()) {
        DataRow row = m_iterator.next();
        // get grouping state according to new row
        m_currentGroupingState = getGroupingState(row);
        groupEnd = m_currentGroupingState.isGroupEnd();
        // to duplicate checker.
        if (m_lastRow == null) {
            m_lastGroupingState = m_currentGroupingState;
            if (checkDuplicates) {
                m_duplicateChecker.addKey(m_currentGroupingState.getGroupIdentifier());
            }
        }
        m_lastRow = row;
        // if group end has not been reached add row
        if (!groupEnd) {
            cont.addRowToTable(row);
            m_lastGroupingState = m_currentGroupingState;
        // if group end has been reached add identifier of new group to
        // duplicate checker
        } else {
            if (checkDuplicates) {
                try {
                    m_duplicateChecker.addKey(m_currentGroupingState.getGroupIdentifier());
                } catch (DuplicateKeyException e) {
                    throw new DuplicateKeyException("Input table was " + "not sorted, found duplicate (group identifier:" + m_currentGroupingState.getGroupIdentifier() + ")");
                }
            }
        }
        // which row will be added.
        if (!m_iterator.hasNext() && !m_isFinalRow) {
            m_isFinalRow = true;
            // thus end loop
            if (!groupEnd) {
                m_endLoop = true;
            }
        }
    }
    cont.close();
    if (m_endLoop) {
        // check for duplicates and throw exception if duplicate exist
        try {
            m_duplicateChecker.checkForDuplicates();
        } catch (DuplicateKeyException e) {
            throw new DuplicateKeyException("Input table was not sorted, found duplicate group identifier " + e.getKey());
        } finally {
            m_duplicateChecker.clear();
            m_duplicateChecker = null;
        }
    }
    // push variables
    pushFlowVariableInt("currentIteration", m_iteration);
    pushGroupColumnValuesAsFlowVariables(m_lastGroupingState);
    pushFlowVariableString("groupIdentifier", m_lastGroupingState.getGroupIdentifier());
    m_iteration++;
    return new BufferedDataTable[] { cont.getTable() };
}