Examples with ExecutionMonitor org.knime.core.node.ExecutionMonitor used on opensource projects

Example 66 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class XValidatePartitionModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    boolean inLoop = (m_partNumbers != null);
    if (!inLoop) {
        if (m_settings.leaveOneOut()) {
            m_nrIterations = inData[0].getRowCount();
            m_currIteration = 0;
            m_partNumbers = new short[0];
        } else {
            m_partNumbers = new short[inData[0].getRowCount()];
            final double partSize = m_partNumbers.length / (double) m_settings.validations();
            if (m_settings.stratifiedSampling()) {
                ExecutionMonitor subExec = exec.createSubProgress(0.0);
                subExec.setMessage("Preparing stratified sampling");
                Map<DataCell, List<Integer>> valueCounts = countValues(inData[0], subExec, m_settings.classColumn());
                int part = 0;
                for (Map.Entry<DataCell, List<Integer>> e : valueCounts.entrySet()) {
                    List<Integer> l = e.getValue();
                    for (Integer i : l) {
                        m_partNumbers[i] = (short) part++;
                        part %= m_settings.validations();
                    }
                }
            } else {
                for (int i = 0; i < m_partNumbers.length; i++) {
                    m_partNumbers[i] = (short) Math.min(i / partSize, m_partNumbers.length);
                }
                if (m_settings.randomSampling()) {
                    long seed = m_settings.useRandomSeed() ? m_settings.randomSeed() : System.currentTimeMillis();
                    Random rand = new Random(seed);
                    for (int i = 0; i < m_partNumbers.length; i++) {
                        int pos = rand.nextInt(m_partNumbers.length);
                        short x = m_partNumbers[pos];
                        m_partNumbers[pos] = m_partNumbers[i];
                        m_partNumbers[i] = x;
                    }
                }
            }
            m_nrIterations = m_settings.validations();
            m_currIteration = 0;
        }
    }
    BufferedDataContainer test = exec.createDataContainer(inData[0].getDataTableSpec());
    BufferedDataContainer train = exec.createDataContainer(inData[0].getDataTableSpec());
    int count = 0;
    final double max = inData[0].getRowCount();
    for (DataRow row : inData[0]) {
        exec.checkCanceled();
        exec.setProgress(count / max);
        if (m_settings.leaveOneOut() && (count == m_currIteration)) {
            test.addRowToTable(row);
        } else if (!m_settings.leaveOneOut() && (m_partNumbers[count] == m_currIteration)) {
            test.addRowToTable(row);
        } else {
            train.addRowToTable(row);
        }
        count++;
    }
    test.close();
    train.close();
    // we need to put the counts on the stack for the loop's tail to see:
    pushFlowVariableInt("currentIteration", m_currIteration);
    pushFlowVariableInt("maxIterations", m_nrIterations);
    m_currIteration++;
    return new BufferedDataTable[] { train.getTable(), test.getTable() };
}

Example 67 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class WorkflowManager method saveAs.

/**
 * Saves the workflow to a new location, setting the argument directory as the new NC dir. It will first copy the
 * "old" directory, point the NC dir to the new location and then do an incremental save.
 *
 * @param newContext the new workflow context, including the changed path
 * @param exec The execution monitor
 * @throws IOException If an IO error occured
 * @throws CanceledExecutionException If the execution was canceled
 * @throws LockFailedException If locking failed
 * @since 3.3
 */
public void saveAs(final WorkflowContext newContext, final ExecutionMonitor exec) throws IOException, CanceledExecutionException, LockFailedException {
    if (this == ROOT) {
        throw new IOException("Can't save root workflow");
    }
    try (WorkflowLock lock = lock()) {
        ReferencedFile ncDirRef = getNodeContainerDirectory();
        if (!isProject()) {
            throw new IOException("Cannot call save-as on a non-project workflow");
        }
        File directory = newContext.getCurrentLocation();
        directory.mkdirs();
        if (!directory.isDirectory() || !directory.canWrite()) {
            throw new IOException("Cannot write to " + directory);
        }
        boolean isNCDirNullOrRootReferenceFolder = ncDirRef == null || ncDirRef.getParent() == null;
        if (!isNCDirNullOrRootReferenceFolder) {
            throw new IOException("Referenced directory pointer is not hierarchical: " + ncDirRef);
        }
        m_workflowContext = newContext;
        ReferencedFile autoSaveDirRef = getAutoSaveDirectory();
        ExecutionMonitor saveExec;
        File ncDir = ncDirRef != null ? ncDirRef.getFile() : null;
        if (!ConvenienceMethods.areEqual(ncDir, directory)) {
            // new location
            // cannot be null
            ncDirRef.writeLock();
            try {
                ExecutionMonitor copyExec = exec.createSubProgress(0.5);
                final String copymsg = String.format("Copying existing workflow to new location " + "(from \"%s\" to \"%s\")", ncDir, directory);
                exec.setMessage(copymsg);
                LOGGER.debug(copymsg);
                copyExec.setProgress(1.0);
                FileUtils.copyDirectory(ncDir, directory, /* all but .knimeLock */
                FileFilterUtils.notFileFilter(FileFilterUtils.nameFileFilter(VMFileLocker.LOCK_FILE, IOCase.SENSITIVE)));
                exec.setMessage("Incremental save");
                ncDirRef.changeRoot(directory);
                if (autoSaveDirRef != null) {
                    File newLoc = WorkflowSaveHelper.getAutoSaveDirectory(ncDirRef);
                    final File autoSaveDir = autoSaveDirRef.getFile();
                    if (autoSaveDir.exists()) {
                        try {
                            FileUtils.moveDirectory(autoSaveDir, newLoc);
                            autoSaveDirRef.changeRoot(newLoc);
                            LOGGER.debugWithFormat("Moved auto-save directory from \"%s\" to \"%s\"", autoSaveDir.getAbsolutePath(), newLoc.getAbsolutePath());
                        } catch (IOException ioe) {
                            LOGGER.error(String.format("Couldn't move auto save directory \"%s\" to \"%s\": %s", autoSaveDir.getAbsolutePath(), newLoc.getAbsolutePath(), ioe.getMessage()), ioe);
                        }
                    } else {
                        autoSaveDirRef.changeRoot(newLoc);
                    }
                }
                m_isWorkflowDirectoryReadonly = false;
            } finally {
                ncDirRef.writeUnlock();
            }
            saveExec = exec.createSubProgress(0.5);
        } else {
            saveExec = exec;
        }
        save(directory, saveExec, true);
    }
}

Example 68 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class WorkflowManager method loadExecutionResult.

/**
 * {@inheritDoc}
 */
@Override
public void loadExecutionResult(final NodeContainerExecutionResult result, final ExecutionMonitor exec, final LoadResult loadResult) {
    CheckUtils.checkArgument(result instanceof WorkflowExecutionResult, "Argument must be instance of \"%s\": %s", WorkflowExecutionResult.class.getSimpleName(), result == null ? "null" : result.getClass().getSimpleName());
    WorkflowExecutionResult r = (WorkflowExecutionResult) result;
    try (WorkflowLock lock = lock()) {
        super.loadExecutionResult(result, exec, loadResult);
        Map<NodeID, NodeContainerExecutionResult> map = r.getExecutionResultMap();
        final int count = map.size();
        // contains the corrected NodeID in this workflow (the node ids in
        // the execution result refer to the base id of the remote workflow)
        Map<NodeID, NodeID> transMap = new HashMap<NodeID, NodeID>();
        NodeID otherIDPrefix = r.getBaseID();
        for (NodeID otherID : map.keySet()) {
            assert otherID.hasSamePrefix(otherIDPrefix);
            transMap.put(new NodeID(getID(), otherID.getIndex()), otherID);
        }
        for (NodeID id : m_workflow.createBreadthFirstSortedList(transMap.keySet(), true).keySet()) {
            NodeID otherID = transMap.get(id);
            NodeContainer nc = m_workflow.getNode(id);
            NodeContainerExecutionResult exResult = map.get(otherID);
            if (exResult == null) {
                loadResult.addError("No execution result for node " + nc.getNameWithID());
                continue;
            }
            exec.setMessage(nc.getNameWithID());
            ExecutionMonitor subExec = exec.createSubProgress(1.0 / count);
            // Propagagte the flow variables
            if (nc instanceof SingleNodeContainer) {
                NodeOutPort[] predecessorOutPorts = assemblePredecessorOutPorts(id);
                FlowObjectStack[] sos = Arrays.stream(predecessorOutPorts).map(p -> p != null ? p.getFlowObjectStack() : null).toArray(FlowObjectStack[]::new);
                createAndSetFlowObjectStackFor((SingleNodeContainer) nc, sos);
            }
            nc.loadExecutionResult(exResult, subExec, loadResult);
            subExec.setProgress(1.0);
        }
    }
}

Example 69 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class WorkflowManager method loadMetaNodeTemplate.

/**
 * Reads the template info from the metanode argument and then resolves that URI and returns a workflow manager that
 * lives as child of {@link #templateWorkflowRoot}. Used to avoid duplicate loading from a remote location. The
 * returned instance is then copied to the final destination.
 */
private NodeContainerTemplate loadMetaNodeTemplate(final NodeContainerTemplate meta, final WorkflowLoadHelper loadHelper, final LoadResult loadResult) throws IOException, UnsupportedWorkflowVersionException, CanceledExecutionException {
    MetaNodeTemplateInformation linkInfo = meta.getTemplateInformation();
    URI sourceURI = linkInfo.getSourceURI();
    WorkflowManager tempParent = lazyInitTemplateWorkflowRoot();
    MetaNodeLinkUpdateResult loadResultChild;
    NodeContext.pushContext((NodeContainer) meta);
    try {
        if (m_workflowContext != null && m_workflowContext.getMountpointURI().isPresent() && sourceURI.getHost().startsWith("knime.") && (ResolverUtil.resolveURItoLocalFile(m_workflowContext.getMountpointURI().get()) == null)) {
            // a workflow relative template URI but the workflow itself is not local
            // => the template is also not local and must be resolved using the workflow's original location
            URI origWfUri = m_workflowContext.getMountpointURI().get();
            String combinedPath = origWfUri.getPath() + sourceURI.getPath();
            sourceURI = new URI(origWfUri.getScheme(), origWfUri.getUserInfo(), origWfUri.getHost(), origWfUri.getPort(), combinedPath, origWfUri.getQuery(), origWfUri.getFragment()).normalize();
        }
        File localDir = ResolverUtil.resolveURItoLocalOrTempFile(sourceURI);
        if (localDir.isFile()) {
            // looks like a zipped metanode downloaded from a 4.4+ server
            File unzipped = FileUtil.createTempDir("metanode-template");
            FileUtil.unzip(localDir, unzipped);
            localDir = unzipped.listFiles()[0];
        }
        TemplateNodeContainerPersistor loadPersistor = loadHelper.createTemplateLoadPersistor(localDir, sourceURI);
        loadResultChild = new MetaNodeLinkUpdateResult("Template from " + sourceURI.toString());
        tempParent.load(loadPersistor, loadResultChild, new ExecutionMonitor(), false);
    } catch (InvalidSettingsException | URISyntaxException e) {
        throw new IOException("Unable to read template metanode: " + e.getMessage(), e);
    } finally {
        NodeContext.removeLastContext();
    }
    NodeContainerTemplate linkResult = loadResultChild.getLoadedInstance();
    MetaNodeTemplateInformation templInfo = linkResult.getTemplateInformation();
    Role sourceRole = templInfo.getRole();
    switch(sourceRole) {
        case Link:
            // (this is due to the template information link uri set above)
            break;
        default:
            throw new IOException("The source of the linked instance does " + "not represent a template but is of role " + sourceRole);
    }
    loadResult.addChildError(loadResultChild);
    return linkResult;
}

Example 70 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class HiliteScorerNodeModel method execute.

/**
 * Starts the scoring in the scorer.
 *
 * @param data
 *            the input data of length one
 * @param exec
 *            the execution monitor
 * @return the confusion matrix
 * @throws CanceledExecutionException
 *             if user canceled execution
 *
 * @see NodeModel#execute(BufferedDataTable[],ExecutionContext)
 */
@SuppressWarnings("unchecked")
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] data, final ExecutionContext exec) throws CanceledExecutionException {
    // check input data
    assert (data != null && data.length == 1 && data[INPORT] != null);
    // blow away result from last execute (should have been reset anyway)
    // first try to figure out what are the different class values
    // in the two respective columns
    BufferedDataTable in = data[INPORT];
    DataTableSpec inSpec = in.getDataTableSpec();
    final int index1 = inSpec.findColumnIndex(m_firstCompareColumn);
    final int index2 = inSpec.findColumnIndex(m_secondCompareColumn);
    // two elements, first is column names, second row names;
    // these arrays are ordered already, i.e. if both columns have
    // cells in common (e.g. both have Iris-Setosa), they get the same
    // index in the array. thus, the high numbers should appear
    // in the diagonal
    DataCell[] values = determineColValues(in, index1, index2, exec.createSubProgress(0.5));
    List<DataCell> valuesList = Arrays.asList(values);
    Set<DataCell> valuesInCol2 = new HashSet<DataCell>();
    m_correctCount = 0;
    m_falseCount = 0;
    // the key store remembers the row key for later hiliting
    List[][] keyStore = new List[values.length][values.length];
    // the scorerCount counts the confusions
    m_scorerCount = new int[values.length][values.length];
    // init the matrix
    for (int i = 0; i < keyStore.length; i++) {
        for (int j = 0; j < keyStore[i].length; j++) {
            keyStore[i][j] = new ArrayList<RowKey>();
        }
    }
    int rowCnt = in.getRowCount();
    int rowNr = 0;
    ExecutionMonitor subExec = exec.createSubProgress(0.5);
    for (Iterator<DataRow> it = in.iterator(); it.hasNext(); rowNr++) {
        DataRow row = it.next();
        subExec.setProgress((1.0 + rowNr) / rowCnt, "Computing score, row " + rowNr + " (\"" + row.getKey() + "\") of " + in.getRowCount());
        try {
            subExec.checkCanceled();
        } catch (CanceledExecutionException cee) {
            reset();
            throw cee;
        }
        DataCell cell1 = row.getCell(index1);
        DataCell cell2 = row.getCell(index2);
        valuesInCol2.add(cell2);
        if (cell1.isMissing() || cell2.isMissing()) {
            continue;
        }
        boolean areEqual = cell1.equals(cell2);
        int i1 = valuesList.indexOf(cell1);
        int i2 = areEqual ? i1 : valuesList.indexOf(cell2);
        assert i1 >= 0 : "column spec lacks possible value " + cell1;
        assert i2 >= 0 : "column spec lacks possible value " + cell2;
        // i2 must be equal to i1 if cells are equal (implication)
        assert (!areEqual || i1 == valuesList.indexOf(cell2));
        keyStore[i1][i2].add(row.getKey());
        m_scorerCount[i1][i2]++;
        if (areEqual) {
            m_correctCount++;
        } else {
            m_falseCount++;
        }
    }
    m_nrRows = rowNr;
    HashSet<String> valuesAsStringSet = new HashSet<String>();
    HashSet<String> duplicateValuesAsString = new HashSet<String>();
    for (DataCell c : values) {
        valuesAsStringSet.add(c.toString());
    }
    for (DataCell c : values) {
        String cAsString = c.toString();
        if (!valuesAsStringSet.remove(cAsString)) {
            duplicateValuesAsString.add(cAsString);
        }
    }
    boolean hasPrintedWarningOnAmbiguousValues = false;
    m_values = new String[values.length];
    for (int i = 0; i < m_values.length; i++) {
        DataCell c = values[i];
        String s = c.toString();
        if (duplicateValuesAsString.contains(s)) {
            boolean isInSecondColumn = valuesInCol2.contains(c);
            int uniquifier = 1;
            if (isInSecondColumn) {
                s = s.concat(" (" + m_secondCompareColumn + ")");
            } else {
                s = s.concat(" (" + m_firstCompareColumn + ")");
            }
            String newName = s;
            while (!valuesAsStringSet.add(newName)) {
                newName = s + "#" + (uniquifier++);
            }
            m_values[i] = newName;
            if (!hasPrintedWarningOnAmbiguousValues) {
                hasPrintedWarningOnAmbiguousValues = true;
                setWarningMessage("Ambiguous value \"" + c.toString() + "\" encountered. Preserving individual instances;" + " consider to convert input columns to string");
            }
        } else {
            int uniquifier = 1;
            String newName = s;
            while (!valuesAsStringSet.add(newName)) {
                newName = s + "#" + (uniquifier++);
            }
            m_values[i] = newName;
        }
    }
    DataType[] colTypes = new DataType[m_values.length];
    Arrays.fill(colTypes, IntCell.TYPE);
    BufferedDataContainer container = exec.createDataContainer(new DataTableSpec(m_values, colTypes));
    for (int i = 0; i < m_values.length; i++) {
        // need to make a datacell for the row key
        container.addRowToTable(new DefaultRow(m_values[i], m_scorerCount[i]));
    }
    container.close();
    // print info
    int correct = getCorrectCount();
    int incorrect = getFalseCount();
    double error = getError();
    int nrRows = getNrRows();
    int missing = nrRows - correct - incorrect;
    m_keyStore = keyStore;
    LOGGER.info("error=" + error + ", #correct=" + correct + ", #false=" + incorrect + ", #rows=" + nrRows + ", #missing=" + missing);
    // our view displays the table - we must keep a reference in the model.
    BufferedDataTable result = container.getTable();
    return new BufferedDataTable[] { result };
}