Examples with ExecutionContext org.knime.core.node.ExecutionContext used on opensource projects

Example 96 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class SortedCorrelationComputer method calculateSpearman.

/**
 * Calculates the Spearmans rank for all pairs of Data table columns based on previously calculated ranks.
 * @param exec the Execution context.
 * @return the output matrix to be turned into the output model
 * @throws CanceledExecutionException if canceled by users
 */
protected HalfDoubleMatrix calculateSpearman(final ExecutionContext exec) throws CanceledExecutionException {
    // // the ranking must have been calculated before
    assert (m_rank != null);
    double progStep = 0.5;
    CorrelationComputer calculator = new CorrelationComputer(m_rank.getDataTableSpec(), 0);
    exec.setMessage("Calculating table statistics");
    ExecutionContext execStep1 = exec.createSubExecutionContext(progStep);
    calculator.calculateStatistics(m_rank, execStep1);
    execStep1.setProgress(1.0);
    exec.setMessage("Calculating correlation values");
    ExecutionMonitor execStep2 = exec.createSubExecutionContext(progStep);
    return calculator.calculateOutput(m_rank, execStep2);
}

Example 97 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class CronbachNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
    final BufferedDataTable in = (BufferedDataTable) inData[0];
    final DataTableSpec inSpec = in.getDataTableSpec();
    ColumnRearranger filteredTableRearranger = new ColumnRearranger(inSpec);
    String[] includeNames = m_columnFilterModel.applyTo(inSpec).getIncludes();
    filteredTableRearranger.keepOnly(includeNames);
    final BufferedDataTable filteredTable = exec.createColumnRearrangeTable(in, filteredTableRearranger, exec.createSilentSubExecutionContext(0.0));
    final DataTableSpec filteredTableSpec = filteredTable.getDataTableSpec();
    // step1 get variance for all columns
    Variance my = new Variance(filteredTableSpec.getColumnNames());
    StatisticCalculator sc = new StatisticCalculator(filteredTableSpec, my);
    sc.evaluate(filteredTable, exec.createSubExecutionContext(0.5));
    double[] sum = new double[filteredTable.getRowCount()];
    // step2 get variance for the overall sum
    ExecutionContext exec2 = exec.createSubExecutionContext(0.5);
    int rowCount = filteredTable.getRowCount();
    int i = 0;
    for (DataRow row : filteredTable) {
        sum[i] = 0;
        exec2.checkCanceled();
        exec2.setProgress(i * 1.0 / rowCount, "Statisics calculation row " + i + " of " + rowCount);
        for (DataCell cell : row) {
            if (!cell.isMissing()) {
                double value = ((DoubleValue) cell).getDoubleValue();
                sum[i] += value;
            } else {
                throw new InvalidSettingsException("Missing Values are not supported. " + "Please resolve them with the Missing Value node.");
            }
        }
        i++;
    }
    exec.setMessage("Caluating Crombach over all Columns");
    double cronbach = 0;
    for (String s : filteredTableSpec.getColumnNames()) {
        cronbach += my.getResult(s);
        exec.checkCanceled();
    }
    org.apache.commons.math3.stat.descriptive.moment.Variance v = new org.apache.commons.math3.stat.descriptive.moment.Variance();
    cronbach /= v.evaluate(sum);
    double k = filteredTableSpec.getNumColumns();
    cronbach = k / (k - 1) * (1.0 - cronbach);
    BufferedDataContainer out = exec.createDataContainer(getDataTableSpec());
    if (in.getRowCount() <= 0) {
        setWarningMessage("Empty input table, no value calculated!");
    }
    DataRow r = new DefaultRow(new RowKey("Cronbach"), new DoubleCell(cronbach));
    out.addRowToTable(r);
    out.close();
    return new BufferedDataTable[] { out.getTable() };
}

Example 98 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class StringToDurationPeriodNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        private SimpleStreamableOperatorInternals m_internals = new SimpleStreamableOperatorInternals();

        /**
         * {@inheritDoc}
         */
        @Override
        public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
            final RowInput rowInput = (RowInput) inputs[0];
            final DataRow row = rowInput.poll();
            if (row != null) {
                final DataTableSpec inSpec = rowInput.getDataTableSpec();
                final int[] includeIndexes = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
                final Config config = m_internals.getConfig();
                // detect types
                detectTypes(new OneRowAdditionalRowInput(rowInput, row));
                for (int i = 0; i < m_detectedTypes.length; i++) {
                    config.addDataType("detected_type" + i, m_detectedTypes[i]);
                }
                // write detected types and column names into config
                if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                    for (int i = 0; i < rowInput.getDataTableSpec().getNumColumns(); i++) {
                        final int searchIdx = Arrays.binarySearch(includeIndexes, i);
                        config.addString("colname" + i, inSpec.getColumnNames()[i]);
                        if (searchIdx < 0) {
                            config.addDataType("type" + i, inSpec.getColumnSpec(i).getType());
                        } else {
                            config.addDataType("type" + i, m_detectedTypes[searchIdx] != null ? m_detectedTypes[searchIdx] : null);
                        }
                    }
                    config.addInt("sizeRow", rowInput.getDataTableSpec().getNumColumns());
                } else {
                    for (int i = 0; i < inSpec.getNumColumns(); i++) {
                        config.addString("colname" + i, inSpec.getColumnNames()[i]);
                        config.addDataType("type" + i, inSpec.getColumnSpec(i).getType());
                    }
                    for (int i = 0; i < m_detectedTypes.length; i++) {
                        config.addString("colname" + (i + inSpec.getNumColumns()), new UniqueNameGenerator(inSpec).newName(inSpec.getColumnSpec(includeIndexes[i]).getName() + m_suffix.getStringValue()));
                        config.addDataType("type" + (i + inSpec.getNumColumns()), m_detectedTypes[i]);
                    }
                    config.addInt("sizeRow", inSpec.getNumColumns() + m_detectedTypes.length);
                }
                config.addBoolean("needsIteration", false);
            } else {
                m_internals.getConfig().addInt("sizeRow", 0);
            }
            rowInput.close();
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public StreamableOperatorInternals saveInternals() {
            return m_internals;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void loadInternals(final StreamableOperatorInternals internals) {
            m_internals = (SimpleStreamableOperatorInternals) internals;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            final RowInput in = (RowInput) inputs[0];
            final RowOutput out = (RowOutput) outputs[0];
            final DataTableSpec inSpec = in.getDataTableSpec();
            final int[] includeIndexes = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
            // read detected types from config
            final DataType[] detectedTypes = new DataType[includeIndexes.length];
            final Config config = m_internals.getConfig();
            for (int i = 0; i < includeIndexes.length; i++) {
                detectedTypes[i] = config.getDataType("detected_type" + i, null);
            }
            // compute every row
            DataRow row;
            while ((row = in.poll()) != null) {
                exec.checkCanceled();
                DataCell[] datacells = new DataCell[includeIndexes.length];
                for (int i = 0; i < includeIndexes.length; i++) {
                    if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                        final StringToDurationPeriodCellFactory cellFac = new StringToDurationPeriodCellFactory(new DataColumnSpecCreator(inSpec.getColumnNames()[includeIndexes[i]], detectedTypes[i]).createSpec(), includeIndexes[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    } else {
                        final DataColumnSpec dataColSpec = new UniqueNameGenerator(inSpec).newColumn(inSpec.getColumnNames()[includeIndexes[i]] + m_suffix.getStringValue(), detectedTypes[i]);
                        final StringToDurationPeriodCellFactory cellFac = new StringToDurationPeriodCellFactory(dataColSpec, includeIndexes[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    }
                }
                if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                    out.push(new ReplacedColumnsDataRow(row, datacells, includeIndexes));
                } else {
                    out.push(new AppendedColumnRow(row, datacells));
                }
            }
            in.close();
            out.close();
        }
    };
}

Example 99 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class ExtendedStatisticsNodeModel method execute.

/**
 * {@inheritDoc}
 *
 * @throws CanceledExecutionException
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws CanceledExecutionException {
    double initPercent = m_enableHiLite.getBooleanValue() ? .25 : .2;
    ExecutionContext init = exec.createSubExecutionContext(initPercent);
    DataTableSpec dataSpec = inData[0].getDataTableSpec();
    List<String> includes = nominalColumns(dataSpec);
    m_statTable = new Statistics3Table(inData[0], m_computeMedian.getBooleanValue(), numOfNominalValuesOutput(), includes, init);
    if (getStatTable().getWarning() != null) {
        setWarningMessage(getStatTable().getWarning());
    }
    BufferedDataTable outTableOccurrences = exec.createBufferedDataTable(getStatTable().createNominalValueTable(includes), exec.createSubProgress(0.5));
    BufferedDataTable[] ret = new BufferedDataTable[3];
    DataTableSpec newSpec = renamedOccurrencesSpec(outTableOccurrences.getSpec());
    ret[2] = exec.createSpecReplacerTable(outTableOccurrences, newSpec);
    ExecutionContext table = exec.createSubExecutionContext(initPercent);
    ret[0] = getStatTable().createStatisticsInColumnsTable(table);
    ExecutionContext histogram = exec.createSubExecutionContext(1.0 / 2);
    final HistogramColumn histogramColumn = createHistogramColumn();
    HiLiteHandler hlHandler = getEnableHiLite().getBooleanValue() ? getInHiLiteHandler(0) : new HiLiteHandler();
    double[] mins = getStatTable().getMin(), maxes = getStatTable().getMax(), means = getStatTable().getMean();
    for (int i = 0; i < maxes.length; i++) {
        DataCell min = getStatTable().getNonInfMin(i);
        if (min.isMissing()) {
            mins[i] = Double.NaN;
        } else {
            mins[i] = ((DoubleValue) min).getDoubleValue();
        }
        DataCell max = getStatTable().getNonInfMax(i);
        if (max.isMissing()) {
            maxes[i] = Double.NaN;
        } else {
            maxes[i] = ((DoubleValue) max).getDoubleValue();
        }
    }
    Pair<BufferedDataTable, Map<Integer, ? extends HistogramModel<?>>> pair = histogramColumn.process(histogram, inData[0], hlHandler, ret[0], mins, maxes, means, numOfNominalValues(), getColumnNames());
    // final BufferedDataTable outTable =
    // histogramColumn.appendNominal(pair.getFirst(), getStatTable(), hlHandler, exec, numOfNominalValues());
    ret[0] = pair.getFirst();
    ret[1] = histogramColumn.nominalTable(getStatTable(), hlHandler, exec, numOfNominalValues());
    if (m_enableHiLite.getBooleanValue()) {
        double rest = 1 - initPercent * 2 - 1.0 / 2;
        ExecutionContext projection = exec.createSubExecutionContext(rest / 2);
        ColumnRearranger rearranger = new ColumnRearranger(dataSpec);
        Set<String> colNames = new HashSet<String>(Arrays.asList(getColumnNames()));
        for (DataColumnSpec spec : rearranger.createSpec()) {
            if ((!spec.getType().isCompatible(DoubleValue.class) && !spec.getType().isCompatible(NominalValue.class)) || !colNames.contains(spec.getName())) {
                rearranger.remove(spec.getName());
            }
        }
        ExecutionContext save = exec.createSubExecutionContext(rest / 2);
        m_subTable = new DefaultDataArray(projection.createColumnRearrangeTable(inData[0], rearranger, projection), 1, inData[0].getRowCount(), save);
        m_histograms = histogramColumn.histograms(inData[0], getInHiLiteHandler(0), mins, maxes, means, getColumnNames());
        Set<String> nominalColumns = new LinkedHashSet<String>();
        for (int i = 0; i < inData[0].getSpec().getNumColumns(); ++i) {
            Map<DataCell, Integer> nominalValues = getStatTable().getNominalValues(i);
            if (nominalValues != null) {
                nominalColumns.add(inData[0].getSpec().getColumnSpec(i).getName());
            }
        }
        final Pair<Map<Integer, Map<Integer, Set<RowKey>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> bucketsAndNominals = HistogramColumn.construct(m_histograms, m_subTable, nominalColumns);
        m_buckets = bucketsAndNominals.getFirst();
        m_nominalKeys = bucketsAndNominals.getSecond();
    } else {
        m_histograms = pair.getSecond();
    }
    return ret;
}

Example 100 with ExecutionContext

use of org.knime.core.node.ExecutionContext in project knime-core by knime.

the class OldToNewTimeNodeModel method createStreamableOperator.

/**
 * {@inheritDoc}
 */
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
    return new StreamableOperator() {

        SimpleStreamableOperatorInternals m_internals = new SimpleStreamableOperatorInternals();

        /**
         * {@inheritDoc}
         */
        @Override
        public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
            if (partitionInfo.getPartitionIndex() == 0) {
                final RowInput rowInput = (RowInput) inputs[0];
                final DataRow row = rowInput.poll();
                if (row != null) {
                    if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                        final DataColumnSpec[] colSpecs = new DataColumnSpec[row.getNumCells()];
                        final DataTableSpec inSpec = rowInput.getDataTableSpec();
                        final DataColumnSpec[] newColumnSpecs = getNewIncludedColumnSpecs(inSpec, row);
                        final int[] includeIndexes = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
                        for (int i = 0; i < inSpec.getNumColumns(); i++) {
                            final int searchIdx = Arrays.binarySearch(includeIndexes, i);
                            if (searchIdx < 0) {
                                colSpecs[i] = inSpec.getColumnSpec(i);
                            } else {
                                colSpecs[i] = newColumnSpecs[searchIdx];
                            }
                        }
                        final Config config = m_internals.getConfig();
                        config.addBoolean("hasIterated", false);
                        for (int i = 0; i < inSpec.getNumColumns(); i++) {
                            config.addDataType("type" + i, colSpecs[i].getType());
                            config.addString("colname" + i, colSpecs[i].getName());
                        }
                        config.addInt("sizeRow", colSpecs.length);
                    } else {
                        final DataTableSpec inSpec = rowInput.getDataTableSpec();
                        final DataColumnSpec[] newColumnSpecs = getNewIncludedColumnSpecs(inSpec, row);
                        final int[] includeIndexes = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
                        final DataColumnSpec[] colSpecs = new DataColumnSpec[row.getNumCells() + includeIndexes.length];
                        for (int i = 0; i < inSpec.getNumColumns(); i++) {
                            colSpecs[i] = inSpec.getColumnSpec(i);
                        }
                        for (int i = 0; i < newColumnSpecs.length; i++) {
                            colSpecs[i + inSpec.getNumColumns()] = new UniqueNameGenerator(inSpec).newColumn(newColumnSpecs[i].getName() + m_suffix.getStringValue(), newColumnSpecs[i].getType());
                        }
                        final Config config = m_internals.getConfig();
                        config.addBoolean("hasIterated", false);
                        for (int i = 0; i < colSpecs.length; i++) {
                            config.addDataType("type" + i, colSpecs[i].getType());
                            config.addString("colname" + i, colSpecs[i].getName());
                        }
                        config.addInt("sizeRow", colSpecs.length);
                    }
                } else {
                    m_internals.getConfig().addInt("sizeRow", 0);
                }
            }
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public StreamableOperatorInternals saveInternals() {
            return m_internals;
        }

        @Override
        public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
            final RowInput in = (RowInput) inputs[0];
            final RowOutput out = (RowOutput) outputs[0];
            final DataTableSpec inSpec = in.getDataTableSpec();
            final int[] includeIndexes = Arrays.stream(m_colSelect.applyTo(inSpec).getIncludes()).mapToInt(s -> inSpec.findColumnIndex(s)).toArray();
            DataRow row;
            while ((row = in.poll()) != null) {
                exec.checkCanceled();
                final DataColumnSpec[] newColumnSpecs = getNewIncludedColumnSpecs(inSpec, row);
                DataCell[] datacells = new DataCell[includeIndexes.length];
                for (int i = 0; i < includeIndexes.length; i++) {
                    if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                        ConvertTimeCellFactory cellFac = new ConvertTimeCellFactory(newColumnSpecs[i], i, includeIndexes[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    } else {
                        final DataColumnSpec dataColSpec = new UniqueNameGenerator(inSpec).newColumn(newColumnSpecs[i].getName() + m_suffix.getStringValue(), newColumnSpecs[i].getType());
                        ConvertTimeCellFactory cellFac = new ConvertTimeCellFactory(dataColSpec, i, includeIndexes[i]);
                        datacells[i] = cellFac.getCells(row)[0];
                    }
                }
                if (m_isReplaceOrAppend.getStringValue().equals(OPTION_REPLACE)) {
                    out.push(new ReplacedColumnsDataRow(row, datacells, includeIndexes));
                } else {
                    out.push(new AppendedColumnRow(row, datacells));
                }
            }
            in.close();
            out.close();
        }
    };
}