Examples with ExecutionContext org.knime.core.node.ExecutionContext used on opensource projects

Example 56 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class DBWriterNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            exec.setProgress("Opening database connection to write data...");
            DatabaseConnectionSettings connSettings;
            PortObject portObj = ((PortObjectInput) inputs[1]).getPortObject();
            if (portObj != null && (portObj instanceof DatabaseConnectionPortObject)) {
                connSettings = ((DatabaseConnectionPortObject) portObj).getConnectionSettings(getCredentialsProvider());
            } else {
                connSettings = m_conn;
            }
            DBWriter writer = connSettings.getUtility().getWriter(connSettings);
            // write entire data
            m_errorMessage = writer.writeData(m_tableName, (RowInput) inputs[0], -1, m_append, exec, m_types, getCredentialsProvider(), m_batchSize, m_insertNullForMissingCols);
        }
    };
}

Example 57 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class Normalizer2NodeModel method calculate.

/**
 * New normalized {@link org.knime.core.data.DataTable} is created depending
 * on the mode.
 */
/**
 * @param inData The input data.
 * @param exec For BufferedDataTable creation and progress.
 * @return the result of the calculation
 * @throws Exception If the node calculation fails for any reason.
 */
protected CalculationResult calculate(final PortObject[] inData, final ExecutionContext exec) throws Exception {
    BufferedDataTable inTable = (BufferedDataTable) inData[0];
    DataTableSpec inSpec = inTable.getSpec();
    // extract selected numeric columns
    updateNumericColumnSelection(inSpec);
    Normalizer2 ntable = new Normalizer2(inTable, m_columns);
    long rowcount = inTable.size();
    ExecutionContext prepareExec = exec.createSubExecutionContext(0.3);
    AffineTransTable outTable;
    boolean fixDomainBounds = false;
    switch(m_mode) {
        case NONORM_MODE:
            return new CalculationResult(inTable, new DataTableSpec(), new AffineTransConfiguration());
        case MINMAX_MODE:
            fixDomainBounds = true;
            outTable = ntable.doMinMaxNorm(m_max, m_min, prepareExec);
            break;
        case ZSCORE_MODE:
            outTable = ntable.doZScoreNorm(prepareExec);
            break;
        case DECIMALSCALING_MODE:
            outTable = ntable.doDecimalScaling(prepareExec);
            break;
        default:
            throw new Exception("No mode set");
    }
    if (outTable.getErrorMessage() != null) {
        // something went wrong, report and throw an exception
        throw new Exception(outTable.getErrorMessage());
    }
    if (ntable.getErrorMessage() != null) {
        // something went wrong during initialization, report.
        setWarningMessage(ntable.getErrorMessage());
    }
    DataTableSpec modelSpec = FilterColumnTable.createFilterTableSpec(inSpec, m_columns);
    AffineTransConfiguration configuration = outTable.getConfiguration();
    DataTableSpec spec = outTable.getDataTableSpec();
    // the same transformation, which is not guaranteed to snap to min/max)
    if (fixDomainBounds) {
        DataColumnSpec[] newColSpecs = new DataColumnSpec[spec.getNumColumns()];
        for (int i = 0; i < newColSpecs.length; i++) {
            newColSpecs[i] = spec.getColumnSpec(i);
        }
        for (int i = 0; i < m_columns.length; i++) {
            int index = spec.findColumnIndex(m_columns[i]);
            DataColumnSpecCreator creator = new DataColumnSpecCreator(newColSpecs[index]);
            DataColumnDomainCreator domCreator = new DataColumnDomainCreator(newColSpecs[index].getDomain());
            domCreator.setLowerBound(new DoubleCell(m_min));
            domCreator.setUpperBound(new DoubleCell(m_max));
            creator.setDomain(domCreator.createDomain());
            newColSpecs[index] = creator.createSpec();
        }
        spec = new DataTableSpec(spec.getName(), newColSpecs);
    }
    ExecutionMonitor normExec = exec.createSubProgress(.7);
    BufferedDataContainer container = exec.createDataContainer(spec);
    long count = 1;
    for (DataRow row : outTable) {
        normExec.checkCanceled();
        normExec.setProgress(count / (double) rowcount, "Normalizing row no. " + count + " of " + rowcount + " (\"" + row.getKey() + "\")");
        container.addRowToTable(row);
        count++;
    }
    container.close();
    return new CalculationResult(container.getTable(), modelSpec, configuration);
}

Example 58 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class AutoBinner method execute.

/**
 * Determine bins.
 *
 * @param data the input data
 * @param exec the execution context
 * @return the operation with the discretisation information
 * @throws Exception
 */
public PMMLPreprocDiscretize execute(final BufferedDataTable data, final ExecutionContext exec) throws Exception {
    // Auto configuration when target is not set
    final DataTableSpec spec = data.getDataTableSpec();
    if (null == m_settings.getTargetColumn() || m_settings.getIncludeAll()) {
        addAllNumericCols(spec);
    }
    // determine intervals
    if (m_settings.getMethod().equals(Method.fixedNumber)) {
        BufferedDataTable inData = calcDomainBoundsIfNeccessary(data, exec.createSubExecutionContext(0.9), Arrays.asList(m_settings.getTargetColumn()));
        init(inData.getDataTableSpec());
        Map<String, double[]> edgesMap = new HashMap<String, double[]>();
        for (String target : m_settings.getTargetColumn()) {
            DataTableSpec inSpec = inData.getDataTableSpec();
            DataColumnSpec targetCol = inSpec.getColumnSpec(target);
            // bounds of the domain
            double min = ((DoubleValue) targetCol.getDomain().getLowerBound()).getDoubleValue();
            double max = ((DoubleValue) targetCol.getDomain().getUpperBound()).getDoubleValue();
            // the edges of the bins
            double[] edges = new double[m_settings.getBinCount() + 1];
            edges[0] = min;
            edges[edges.length - 1] = max;
            for (int i = 1; i < edges.length - 1; i++) {
                edges[i] = min + i / (double) m_settings.getBinCount() * (max - min);
            }
            edgesMap.put(target, edges);
        }
        return createDisretizeOp(edgesMap);
    } else if (m_settings.getMethod().equals(Method.sampleQuantiles)) {
        init(spec);
        Map<String, double[]> edgesMap = new LinkedHashMap<String, double[]>();
        final int colCount = m_settings.getTargetColumn().length;
        // contains all numeric columns if include all is set!
        for (String target : m_settings.getTargetColumn()) {
            exec.setMessage("Calculating quantiles (column \"" + target + "\")");
            ExecutionContext colSortContext = exec.createSubExecutionContext(0.7 / colCount);
            ExecutionContext colCalcContext = exec.createSubExecutionContext(0.3 / colCount);
            ColumnRearranger singleRearranger = new ColumnRearranger(spec);
            singleRearranger.keepOnly(target);
            BufferedDataTable singleColSorted = colSortContext.createColumnRearrangeTable(data, singleRearranger, colSortContext);
            SortedTable sorted = new SortedTable(singleColSorted, Collections.singletonList(target), new boolean[] { true }, colSortContext);
            colSortContext.setProgress(1.0);
            double[] edges = createEdgesFromQuantiles(sorted.getBufferedDataTable(), colCalcContext, m_settings.getSampleQuantiles());
            colCalcContext.setProgress(1.0);
            exec.clearTable(singleColSorted);
            edgesMap.put(target, edges);
        }
        return createDisretizeOp(edgesMap);
    } else {
        throw new IllegalStateException("Unknown binning method.");
    }
}

Example 59 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class AutoBinner method execute.

/**
 * Determine bins.
 *
 * @param data the input data
 * @param exec the execution context
 * @return the operation with the discretisation information
 * @throws Exception ...
 */
public PMMLPreprocDiscretize execute(final BufferedDataTable data, final ExecutionContext exec) throws Exception {
    // Auto configuration when target is not set
    final DataTableSpec spec = data.getDataTableSpec();
    if (null == m_settings.getTargetColumn() || m_settings.getIncludeAll()) {
        addAllNumericCols(spec);
    }
    // determine intervals
    if (m_settings.getMethod().equals(Method.fixedNumber)) {
        if (m_settings.getEqualityMethod().equals(EqualityMethod.width)) {
            BufferedDataTable inData = calcDomainBoundsIfNeccessary(data, exec.createSubExecutionContext(0.9), Arrays.asList(m_settings.getTargetColumn()));
            init(inData.getDataTableSpec());
            Map<String, double[]> edgesMap = new HashMap<String, double[]>();
            for (String target : m_settings.getTargetColumn()) {
                DataTableSpec inSpec = inData.getDataTableSpec();
                DataColumnSpec targetCol = inSpec.getColumnSpec(target);
                // bounds of the domain
                double min = ((DoubleValue) targetCol.getDomain().getLowerBound()).getDoubleValue();
                double max = ((DoubleValue) targetCol.getDomain().getUpperBound()).getDoubleValue();
                // the edges of the bins
                double[] edges = new double[m_settings.getBinCount() + 1];
                edges[0] = min;
                edges[edges.length - 1] = max;
                for (int i = 1; i < edges.length - 1; i++) {
                    edges[i] = min + i / (double) m_settings.getBinCount() * (max - min);
                }
                if (m_settings.getIntegerBounds()) {
                    edges = toIntegerBounds(edges);
                }
                edgesMap.put(target, edges);
            }
            return createDisretizeOp(edgesMap);
        } else {
            // EqualityMethod.equalCount
            Map<String, double[]> edgesMap = new HashMap<String, double[]>();
            for (String target : m_settings.getTargetColumn()) {
                int colIndex = data.getDataTableSpec().findColumnIndex(target);
                List<Double> values = new ArrayList<Double>();
                for (DataRow row : data) {
                    if (!row.getCell(colIndex).isMissing()) {
                        values.add(((DoubleValue) row.getCell(colIndex)).getDoubleValue());
                    }
                }
                edgesMap.put(target, findEdgesForEqualCount(values, m_settings.getBinCount()));
            }
            return createDisretizeOp(edgesMap);
        }
    } else if (m_settings.getMethod().equals(Method.sampleQuantiles)) {
        init(spec);
        Map<String, double[]> edgesMap = new LinkedHashMap<String, double[]>();
        final int colCount = m_settings.getTargetColumn().length;
        // contains all numeric columns if include all is set!
        for (String target : m_settings.getTargetColumn()) {
            exec.setMessage("Calculating quantiles (column \"" + target + "\")");
            ExecutionContext colSortContext = exec.createSubExecutionContext(0.7 / colCount);
            ExecutionContext colCalcContext = exec.createSubExecutionContext(0.3 / colCount);
            ColumnRearranger singleRearranger = new ColumnRearranger(spec);
            singleRearranger.keepOnly(target);
            BufferedDataTable singleColSorted = colSortContext.createColumnRearrangeTable(data, singleRearranger, colSortContext);
            SortedTable sorted = new SortedTable(singleColSorted, Collections.singletonList(target), new boolean[] { true }, colSortContext);
            colSortContext.setProgress(1.0);
            double[] edges = createEdgesFromQuantiles(sorted.getBufferedDataTable(), colCalcContext, m_settings.getSampleQuantiles());
            colCalcContext.setProgress(1.0);
            exec.clearTable(singleColSorted);
            if (m_settings.getIntegerBounds()) {
                edges = toIntegerBounds(edges);
            }
            edgesMap.put(target, edges);
        }
        return createDisretizeOp(edgesMap);
    } else {
        throw new IllegalStateException("Unknown binning method.");
    }
}

Example 60 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class ColumnToGridNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    String[] includes = m_configuration.getIncludes();
    String groupColumn = m_configuration.getGroupColumn();
    final ExecutionMonitor mainExec;
    final BufferedDataTable inputTable;
    if (groupColumn != null) {
        exec.setMessage("Sorting input table");
        BufferedDataTable in = inData[0];
        ExecutionContext sortExec = exec.createSubExecutionContext(0.5);
        ColumnRearranger sortFilterRearranger = new ColumnRearranger(in.getDataTableSpec());
        String[] relevantCols = new String[includes.length + 1];
        System.arraycopy(includes, 0, relevantCols, 0, includes.length);
        relevantCols[relevantCols.length - 1] = groupColumn;
        sortFilterRearranger.keepOnly(relevantCols);
        BufferedDataTable toBeSortedTable = exec.createColumnRearrangeTable(in, sortFilterRearranger, exec.createSubProgress(0.0));
        SortedTable sorter = new SortedTable(toBeSortedTable, Collections.singletonList(groupColumn), new boolean[] { true }, sortExec);
        inputTable = sorter.getBufferedDataTable();
        mainExec = exec.createSubProgress(0.5);
    } else {
        inputTable = inData[0];
        mainExec = exec;
    }
    exec.setMessage("Assembling output");
    DataTableSpec spec = inputTable.getDataTableSpec();
    DataTableSpec outSpec = createOutputSpec(spec);
    BufferedDataContainer cont = exec.createDataContainer(outSpec);
    int[] includeIndices = new int[includes.length];
    for (int i = 0; i < includes.length; i++) {
        int index = spec.findColumnIndex(includes[i]);
        includeIndices[i] = index;
    }
    int gridCount = m_configuration.getColCount();
    final int cellCount;
    final int groupColIndex;
    if (groupColumn != null) {
        cellCount = includeIndices.length * gridCount + 1;
        groupColIndex = spec.findColumnIndex(groupColumn);
    } else {
        cellCount = includeIndices.length * gridCount;
        groupColIndex = -1;
    }
    final DataCell[] cells = new DataCell[cellCount];
    PushBackRowIterator it = new PushBackRowIterator(inputTable.iterator());
    long currentRow = 0;
    long totalRows = inputTable.size();
    long currentOutRow = 0;
    DataCell curGroupValue = null;
    while (it.hasNext()) {
        Arrays.fill(cells, DataType.getMissingCell());
        // assign group column (if enabled)
        if (groupColIndex >= 0) {
            DataRow row = it.next();
            curGroupValue = row.getCell(groupColIndex);
            cells[cells.length - 1] = curGroupValue;
            it.pushBack(row);
        }
        for (int grid = 0; grid < gridCount; grid++) {
            if (!it.hasNext()) {
                break;
            }
            DataRow inRow = it.next();
            DataCell groupValue = groupColIndex < 0 ? null : inRow.getCell(groupColIndex);
            if (ConvenienceMethods.areEqual(curGroupValue, groupValue)) {
                mainExec.setProgress(currentRow / (double) totalRows, "Processing row " + currentRow + "/" + totalRows + ": " + inRow.getKey());
                currentRow += 1;
                mainExec.checkCanceled();
                for (int i = 0; i < includeIndices.length; i++) {
                    cells[grid * includeIndices.length + i] = inRow.getCell(includeIndices[i]);
                }
            } else {
                // start new group, i.e. new row
                it.pushBack(inRow);
                break;
            }
        }
        RowKey key = RowKey.createRowKey(currentOutRow++);
        cont.addRowToTable(new DefaultRow(key, cells));
    }
    cont.close();
    return new BufferedDataTable[] { cont.getTable() };
}