Examples with ExecutionMonitor org.knime.core.node.ExecutionMonitor used on opensource projects

Example 91 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class SearchReplaceDictNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    exec.setMessage("Reading dictionary");
    ExecutionMonitor subExec = exec.createSubProgress(0.2);
    m_replacementMap = readDictionary(subExec);
    exec.setMessage("Searching & Replacing");
    DataTableSpec spec = inData[0].getDataTableSpec();
    ColumnRearranger rearranger = createColumnRearranger(spec);
    BufferedDataTable result = exec.createColumnRearrangeTable(inData[0], rearranger, exec.createSubProgress(0.8));
    m_replacementMap = null;
    return new BufferedDataTable[] { result };
}

Example 92 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class AutoHiLiteNodeFactory method createNodeModel.

/**
 * {@inheritDoc}
 */
@Override
public NodeModel createNodeModel() {
    return new NodeModel(1, 1) {

        private SettingsModelBoolean m_smClearHiLites = createClearHilitesModel();

        /**
         * {@inheritDoc}
         */
        @Override
        protected DataTableSpec[] configure(final DataTableSpec[] inSpecs) throws InvalidSettingsException {
            return inSpecs;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
            if (m_smClearHiLites.getBooleanValue()) {
                getInHiLiteHandler(0).fireClearHiLiteEvent();
            }
            final Set<RowKey> keys = new HashSet<RowKey>();
            final HiLiteHandler hlh = getInHiLiteHandler(0);
            long counter = 0;
            long numOfRows = inData[0].size();
            for (final DataRow row : inData[0]) {
                keys.add(row.getKey());
                if (keys.size() == NUMBER_OF_ROWS_HILITED_AT_ONCE) {
                    exec.setProgress(++counter * NUMBER_OF_ROWS_HILITED_AT_ONCE / (double) numOfRows, "HiLiting all rows...");
                    hlh.fireHiLiteEvent(keys);
                    keys.clear();
                }
            }
            hlh.fireHiLiteEvent(keys);
            // wait for hilite to propagate
            ViewUtils.invokeAndWaitInEDT(() -> {
            });
            return inData;
        }

        @Override
        protected void loadInternals(final File nodeInternDir, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
        }

        @Override
        protected void saveInternals(final File nodeInternDir, final ExecutionMonitor exec) throws IOException, CanceledExecutionException {
        }

        @Override
        protected void saveSettingsTo(final NodeSettingsWO settings) {
            m_smClearHiLites.saveSettingsTo(settings);
        }

        @Override
        protected void validateSettings(final NodeSettingsRO settings) throws InvalidSettingsException {
            m_smClearHiLites.validateSettings(settings);
        }

        @Override
        protected void loadValidatedSettingsFrom(final NodeSettingsRO settings) throws InvalidSettingsException {
            m_smClearHiLites.loadSettingsFrom(settings);
        }

        @Override
        protected void reset() {
        // no op
        }
    };
}

Example 93 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class AccuracyScorerNodeModel method execute.

/**
 * Starts the scoring in the scorer.
 *
 * @param data the input data of length one
 * @param exec the execution monitor
 * @return the confusion matrix
 * @throws CanceledExecutionException if user canceled execution
 *
 * @see NodeModel#execute(BufferedDataTable[],ExecutionContext)
 */
@SuppressWarnings("unchecked")
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] data, final ExecutionContext exec) throws CanceledExecutionException {
    // check input data
    assert (data != null && data.length == 1 && data[INPORT] != null);
    // blow away result from last execute (should have been reset anyway)
    // first try to figure out what are the different class values
    // in the two respective columns
    BufferedDataTable in = data[INPORT];
    DataTableSpec inSpec = in.getDataTableSpec();
    final int index1 = inSpec.findColumnIndex(m_firstCompareColumn);
    final int index2 = inSpec.findColumnIndex(m_secondCompareColumn);
    // two elements, first is column names, second row names;
    // these arrays are ordered already, i.e. if both columns have
    // cells in common (e.g. both have Iris-Setosa), they get the same
    // index in the array. thus, the high numbers should appear
    // in the diagonal
    DataCell[] values = determineColValues(in, index1, index2, exec.createSubProgress(0.5));
    List<DataCell> valuesList = Arrays.asList(values);
    Set<DataCell> valuesInCol2 = new HashSet<DataCell>();
    // the key store remembers the row key for later hiliting
    List<RowKey>[][] keyStore = new List[values.length][values.length];
    // the scorerCount counts the confusions
    int[][] scorerCount = new int[values.length][values.length];
    // init the matrix
    for (int i = 0; i < keyStore.length; i++) {
        for (int j = 0; j < keyStore[i].length; j++) {
            keyStore[i][j] = new ArrayList<RowKey>();
        }
    }
    long rowCnt = in.size();
    int numberOfRows = 0;
    int correctCount = 0;
    int falseCount = 0;
    int missingCount = 0;
    ExecutionMonitor subExec = exec.createSubProgress(0.5);
    for (Iterator<DataRow> it = in.iterator(); it.hasNext(); numberOfRows++) {
        DataRow row = it.next();
        subExec.setProgress((1.0 + numberOfRows) / rowCnt, "Computing score, row " + numberOfRows + " (\"" + row.getKey() + "\") of " + in.size());
        try {
            subExec.checkCanceled();
        } catch (CanceledExecutionException cee) {
            reset();
            throw cee;
        }
        DataCell cell1 = row.getCell(index1);
        DataCell cell2 = row.getCell(index2);
        valuesInCol2.add(cell2);
        if (cell1.isMissing() || cell2.isMissing()) {
            ++missingCount;
            CheckUtils.checkState(m_ignoreMissingValues, "Missing value in row: " + row.getKey());
            if (m_ignoreMissingValues) {
                continue;
            }
        }
        boolean areEqual = cell1.equals(cell2);
        int i1 = valuesList.indexOf(cell1);
        int i2 = areEqual ? i1 : valuesList.indexOf(cell2);
        assert i1 >= 0 : "column spec lacks possible value " + cell1;
        assert i2 >= 0 : "column spec lacks possible value " + cell2;
        // i2 must be equal to i1 if cells are equal (implication)
        assert (!areEqual || i1 == valuesList.indexOf(cell2));
        keyStore[i1][i2].add(row.getKey());
        scorerCount[i1][i2]++;
        if (areEqual) {
            correctCount++;
        } else {
            falseCount++;
        }
    }
    HashSet<String> valuesAsStringSet = new HashSet<String>();
    HashSet<String> duplicateValuesAsString = new HashSet<String>();
    for (DataCell c : values) {
        valuesAsStringSet.add(c.toString());
    }
    for (DataCell c : values) {
        String cAsString = c.toString();
        if (!valuesAsStringSet.remove(cAsString)) {
            duplicateValuesAsString.add(cAsString);
        }
    }
    boolean hasPrintedWarningOnAmbiguousValues = false;
    String[] targetValues = new String[values.length];
    for (int i = 0; i < targetValues.length; i++) {
        DataCell c = values[i];
        String s = c.toString();
        if (duplicateValuesAsString.contains(s)) {
            boolean isInSecondColumn = valuesInCol2.contains(c);
            int uniquifier = 1;
            if (isInSecondColumn) {
                s = s.concat(" (" + m_secondCompareColumn + ")");
            } else {
                s = s.concat(" (" + m_firstCompareColumn + ")");
            }
            String newName = s;
            while (!valuesAsStringSet.add(newName)) {
                newName = s + "#" + (uniquifier++);
            }
            targetValues[i] = newName;
            if (!hasPrintedWarningOnAmbiguousValues) {
                hasPrintedWarningOnAmbiguousValues = true;
                addWarning("Ambiguous value \"" + c.toString() + "\" encountered. Preserving individual instances;" + " consider to convert input columns to string");
            }
        } else {
            int uniquifier = 1;
            String newName = s;
            while (!valuesAsStringSet.add(newName)) {
                newName = s + "#" + (uniquifier++);
            }
            targetValues[i] = newName;
        }
    }
    if (missingCount > 0) {
        addWarning("There were missing values in the reference or in the prediction class columns.");
    }
    DataType[] colTypes = new DataType[targetValues.length];
    Arrays.fill(colTypes, IntCell.TYPE);
    BufferedDataContainer container = exec.createDataContainer(new DataTableSpec(targetValues, colTypes));
    for (int i = 0; i < targetValues.length; i++) {
        // need to make a datacell for the row key
        container.addRowToTable(new DefaultRow(targetValues[i], scorerCount[i]));
    }
    container.close();
    ScorerViewData viewData = new ScorerViewData(scorerCount, numberOfRows, falseCount, correctCount, m_firstCompareColumn, m_secondCompareColumn, targetValues, keyStore);
    // print info
    int missing = numberOfRows - correctCount - falseCount;
    LOGGER.info("error=" + viewData.getError() + ", #correct=" + viewData.getCorrectCount() + ", #false=" + viewData.getFalseCount() + ", #rows=" + numberOfRows + ", #missing=" + missing);
    // our view displays the table - we must keep a reference in the model.
    BufferedDataTable result = container.getTable();
    // start creating accuracy statistics
    BufferedDataContainer accTable = exec.createDataContainer(new DataTableSpec(QUALITY_MEASURES_SPECS));
    for (int r = 0; r < targetValues.length; r++) {
        // true positives
        int tp = viewData.getTP(r);
        // false positives
        int fp = viewData.getFP(r);
        // true negatives
        int tn = viewData.getTN(r);
        // false negatives
        int fn = viewData.getFN(r);
        // TP / (TP + FN)
        final DataCell sensitivity;
        // TP / (TP + FN)
        DoubleCell recall = null;
        if (tp + fn > 0) {
            recall = new DoubleCell(1.0 * tp / (tp + fn));
            sensitivity = new DoubleCell(1.0 * tp / (tp + fn));
        } else {
            sensitivity = DataType.getMissingCell();
        }
        // TP / (TP + FP)
        DoubleCell prec = null;
        if (tp + fp > 0) {
            prec = new DoubleCell(1.0 * tp / (tp + fp));
        }
        // TN / (TN + FP)
        final DataCell specificity;
        if (tn + fp > 0) {
            specificity = new DoubleCell(1.0 * tn / (tn + fp));
        } else {
            specificity = DataType.getMissingCell();
        }
        // 2 * Prec. * Recall / (Prec. + Recall)
        final DataCell fmeasure;
        if (recall != null && prec != null) {
            fmeasure = new DoubleCell(2.0 * prec.getDoubleValue() * recall.getDoubleValue() / (prec.getDoubleValue() + recall.getDoubleValue()));
        } else {
            fmeasure = DataType.getMissingCell();
        }
        // add complete row for class value to table
        DataRow row = new DefaultRow(new RowKey(targetValues[r]), new DataCell[] { new IntCell(tp), new IntCell(fp), new IntCell(tn), new IntCell(fn), recall == null ? DataType.getMissingCell() : recall, prec == null ? DataType.getMissingCell() : prec, sensitivity, specificity, fmeasure, DataType.getMissingCell(), DataType.getMissingCell() });
        accTable.addRowToTable(row);
    }
    List<String> classIds = Arrays.asList(targetValues);
    RowKey overallID = new RowKey("Overall");
    int uniquifier = 1;
    while (classIds.contains(overallID.getString())) {
        overallID = new RowKey("Overall (#" + (uniquifier++) + ")");
    }
    // append additional row for overall accuracy
    accTable.addRowToTable(new DefaultRow(overallID, new DataCell[] { DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), DataType.getMissingCell(), new DoubleCell(viewData.getAccuracy()), new DoubleCell(viewData.getCohenKappa()) }));
    accTable.close();
    m_viewData = viewData;
    pushFlowVars(false);
    return new BufferedDataTable[] { result, accTable.getTable() };
}

Example 94 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class SubgroupMinerModel2 method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    BufferedDataTable input = inData[0];
    DataTableSpec spec = input.getDataTableSpec();
    ExecutionMonitor exec1 = exec.createSubProgress(0.5);
    ExecutionMonitor exec2 = exec.createSubProgress(0.5);
    Map<Integer, RowKey> tidRowKeyMapping = new HashMap<Integer, RowKey>();
    LinkedList<DataCell> nameMapping = new LinkedList<DataCell>();
    List<BitVectorValue> transactions;
    AtomicInteger maxBitsetLength = new AtomicInteger(0);
    if (spec.getColumnSpec(m_transactionColumn.getStringValue()).getType().isCompatible(BitVectorValue.class)) {
        transactions = preprocess(input, exec1, tidRowKeyMapping, maxBitsetLength);
        List<String> columnstrings = spec.getColumnSpec(m_transactionColumn.getStringValue()).getElementNames();
        for (String s : columnstrings) {
            nameMapping.add(new StringCell(s));
        }
        // fix #2505: use maximum bitset length
        maxBitsetLength.set(Math.max(maxBitsetLength.get(), nameMapping.size()));
    } else if (spec.getColumnSpec(m_transactionColumn.getStringValue()).getType().isCompatible(CollectionDataValue.class)) {
        transactions = preprocessCollCells(input, exec1, nameMapping, tidRowKeyMapping, maxBitsetLength);
    // for the name Mapping is taken care in the preprocessing
    } else {
        // data value.
        throw new IOException("Selected column is not a possible transaction");
    }
    AprioriAlgorithm apriori = AprioriAlgorithmFactory.getAprioriAlgorithm(AprioriAlgorithmFactory.AlgorithmDataStructure.valueOf(m_underlyingStruct.getStringValue()), maxBitsetLength.get(), input.getRowCount());
    LOGGER.debug("support: " + m_minSupport);
    LOGGER.debug(m_minSupport + " start apriori: " + new Date());
    try {
        apriori.findFrequentItemSets(transactions, m_minSupport.getDoubleValue(), m_maxItemSetLength.getIntValue(), FrequentItemSet.Type.valueOf(m_itemSetType.getStringValue()), exec2);
    } catch (OutOfMemoryError oome) {
        throw new OutOfMemoryError("Execution resulted in an out of memory error, " + "please increase the support threshold.");
    }
    LOGGER.debug("ended apriori: " + new Date());
    BufferedDataTable itemSetTable = createOutputTable(spec, exec, apriori, nameMapping);
    return new BufferedDataTable[] { itemSetTable };
}

Example 95 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class PCANodeModel method execute.

/**
 * Performs the PCA.
 *
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
    // remove all non-numeric columns from the input date
    // final DataTable filteredTable =
    // filterNonNumericalColumns(inData[DATA_INPORT]);
    final BufferedDataTable dataTable = (BufferedDataTable) inData[DATA_INPORT];
    if (dataTable.size() == 0) {
        throw new IllegalArgumentException("Input table is empty!");
    }
    if (dataTable.size() == 1) {
        throw new IllegalArgumentException("Input table has only one row!");
    }
    final double[] meanVector = getMeanVector(dataTable, m_inputColumnIndices, false, exec.createSubExecutionContext(0.2));
    final double[][] m = new double[m_inputColumnIndices.length][m_inputColumnIndices.length];
    final int missingValues = getCovarianceMatrix(exec.createSubExecutionContext(0.2), dataTable, m_inputColumnIndices, meanVector, m);
    final Matrix covarianceMatrix = new Matrix(m);
    if (missingValues > 0) {
        if (m_failOnMissingValues.getBooleanValue()) {
            throw new IllegalArgumentException("missing, infinite or impossible values in table");
        }
        setWarningMessage(missingValues + " rows ignored because of missing" + ", infinite or impossible values");
    }
    final ExecutionContext evdContext = exec.createSubExecutionContext(0.2);
    evdContext.setMessage("computing spectral decomposition");
    final EigenvalueDecomposition eig = covarianceMatrix.eig();
    exec.checkCanceled();
    evdContext.setProgress(0.8);
    final double[] evs = EigenValue.extractEVVector(eig);
    m_dimSelection.setEigenValues(evs);
    final int dimensions = m_dimSelection.getNeededDimensions();
    // don't remember these in case input changes
    m_dimSelection.setEigenValues(null);
    // adjust to selected numerical columns
    if (dimensions > m_inputColumnIndices.length || dimensions < 1) {
        throw new IllegalArgumentException("invalid number of dimensions to reduce to: " + dimensions);
    }
    exec.checkCanceled();
    evdContext.setProgress(0.9);
    final Matrix eigenvectors = EigenValue.getSortedEigenVectors(eig.getV().getArray(), evs, dimensions);
    exec.checkCanceled();
    evdContext.setProgress(1);
    exec.checkCanceled();
    final DataColumnSpec[] specs = createAddTableSpec((DataTableSpec) inData[DATA_INPORT].getSpec(), dimensions);
    final CellFactory fac = new CellFactory() {

        @Override
        public DataCell[] getCells(final DataRow row) {
            return convertInputRow(eigenvectors, row, meanVector, m_inputColumnIndices, dimensions, false);
        }

        @Override
        public DataColumnSpec[] getColumnSpecs() {
            return specs;
        }

        @Override
        public void setProgress(final int curRowNr, final int rowCount, final RowKey lastKey, final ExecutionMonitor texec) {
            texec.setProgress(curRowNr / (double) rowCount, "processing " + curRowNr + " of " + rowCount);
        }
    };
    final ColumnRearranger cr = new ColumnRearranger((DataTableSpec) inData[0].getSpec());
    cr.append(fac);
    if (m_removeOriginalCols.getBooleanValue()) {
        cr.remove(m_inputColumnIndices);
    }
    final BufferedDataTable result = exec.createColumnRearrangeTable((BufferedDataTable) inData[0], cr, exec.createSubProgress(0.4));
    final PortObject[] out = new PortObject[1];
    out[DATA_OUTPORT] = result;
    // m_inputColumnNames);
    return out;
}