Examples with ExecutionMonitor org.knime.core.node.ExecutionMonitor used on opensource projects

Example 21 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class SandboxedNodeCreator method createSandbox.

/**
 * Creates that temporary mini workflow that is executed remotely on the cluster/stream executor.
 * The returned value should be {@link SandboxedNode#close()} when done (using try-with-resources). After this
 * method is called no other set-method should be called.
 *
 * @param exec for progress/cancelation
 * @return the index of the node that represents this node (the node to execute) in the temporary mini workflow
 * @throws InvalidSettingsException
 * @throws IOException
 * @throws CanceledExecutionException
 * @throws LockFailedException
 * @throws InterruptedException
 */
public SandboxedNode createSandbox(final ExecutionMonitor exec) throws InvalidSettingsException, IOException, CanceledExecutionException, LockFailedException, InterruptedException {
    exec.setMessage("Creating virtual workflow");
    final WorkflowManager parent = m_nc.getParent();
    // derive workflow context via NodeContext as the parent could only a be a metanode in a metanode...
    final WorkflowContext origContext = NodeContext.getContext().getWorkflowManager().getContext();
    WorkflowContext.Factory ctxFactory;
    // (specifically reading knime://knime.workflow files)
    if (!m_copyDataIntoNewContext) {
        ctxFactory = new WorkflowContext.Factory(origContext);
        if (m_localWorkflowDir != null) {
            ctxFactory.setOriginalLocation(origContext.getCurrentLocation()).setCurrentLocation(m_localWorkflowDir);
        }
    } else if (m_localWorkflowDir != null) {
        ctxFactory = new WorkflowContext.Factory(m_localWorkflowDir);
    } else {
        ctxFactory = new WorkflowContext.Factory(FileUtil.createTempDir("sandbox-" + m_nc.getNameWithID()));
    }
    // We have to use the same location for the temporary files
    ctxFactory.setTempLocation(origContext.getTempLocation());
    origContext.getMountpointURI().ifPresent(u -> ctxFactory.setMountpointURI(u));
    WorkflowCreationHelper creationHelper = new WorkflowCreationHelper();
    creationHelper.setWorkflowContext(ctxFactory.createContext());
    if (!m_copyDataIntoNewContext) {
        creationHelper.setDataHandlers(parent.getGlobalTableRepository(), parent.getFileStoreHandlerRepository());
    }
    WorkflowManager tempWFM = m_rootWFM.createAndAddProject("Sandbox Exec on " + m_nc.getNameWithID(), creationHelper);
    // Add the workflow variables
    List<FlowVariable> workflowVariables = parent.getProjectWFM().getWorkflowVariables();
    tempWFM.addWorkflowVariables(true, workflowVariables.toArray(new FlowVariable[workflowVariables.size()]));
    // update credentials store of the workflow
    CredentialsStore cs = tempWFM.getCredentialsStore();
    workflowVariables.stream().filter(f -> f.getType().equals(FlowVariable.Type.CREDENTIALS)).filter(f -> !cs.contains(f.getName())).forEach(cs::addFromFlowVariable);
    final int inCnt = m_inData.length;
    // port object IDs in static port object map, one entry for
    // each connected input (no value for unconnected optional inputs)
    List<Integer> portObjectRepositoryIDs = new ArrayList<Integer>(inCnt);
    try {
        NodeID[] ins = new NodeID[inCnt];
        for (int i = 0; i < inCnt; i++) {
            final PortObject in = m_inData[i];
            final NodeInPort inPort = m_nc.getInPort(i);
            final PortType portType = inPort.getPortType();
            if (in == null) {
                // unconnected optional input
                CheckUtils.checkState(portType.isOptional(), "No data at port %d, although port is mandatory (port type %s)", i, portType.getName());
                continue;
            }
            int portObjectRepositoryID = PortObjectRepository.add(in);
            portObjectRepositoryIDs.add(portObjectRepositoryID);
            boolean isTable = BufferedDataTable.TYPE.equals(portType);
            NodeID inID = tempWFM.createAndAddNode(isTable ? TABLE_READ_NODE_FACTORY : OBJECT_READ_NODE_FACTORY);
            NodeSettings s = new NodeSettings("temp_data_in");
            tempWFM.saveNodeSettings(inID, s);
            List<FlowVariable> flowVars = getFlowVariablesOnPort(i);
            PortObjectInNodeModel.setInputNodeSettings(s, portObjectRepositoryID, flowVars, m_copyDataIntoNewContext);
            // update credentials store of the workflow
            flowVars.stream().filter(f -> f.getType().equals(FlowVariable.Type.CREDENTIALS)).filter(f -> !cs.contains(f.getName())).forEach(cs::addFromFlowVariable);
            tempWFM.loadNodeSettings(inID, s);
            ins[i] = inID;
        }
        // execute inPort object nodes to store the input data in them
        if (ins.length > 0 && !tempWFM.executeAllAndWaitUntilDoneInterruptibly()) {
            String error = "Unable to execute virtual workflow, status sent to log facilities";
            LOGGER.debug(error + ":");
            LOGGER.debug(tempWFM.toString());
            throw new RuntimeException(error);
        }
        // add the target node to the workflow
        WorkflowCopyContent.Builder content = WorkflowCopyContent.builder();
        content.setNodeIDs(m_nc.getID());
        final NodeID targetNodeID = tempWFM.copyFromAndPasteHere(parent, content.build()).getNodeIDs()[0];
        NodeContainer targetNode = tempWFM.getNodeContainer(targetNodeID);
        // connect target node to inPort object nodes, skipping unconnected (optional) inputs
        IntStream.range(0, inCnt).filter(i -> ins[i] != null).forEach(i -> tempWFM.addConnection(ins[i], 1, targetNodeID, i));
        if (m_forwardConnectionProgressEvents) {
            setupConnectionProgressEventListeners(m_nc, targetNode);
        }
        // copy the existing tables into the (meta) node (e.g. an executed file reader that's necessary
        // for other nodes to execute)
        exec.setMessage("Copying tables into temp flow");
        NodeContainerExecutionResult origResult = m_nc.createExecutionResult(exec);
        ExecutionMonitor copyExec = exec.createSubProgress(0.0);
        copyExistingTablesIntoSandboxContainer(origResult, m_nc, targetNode, copyExec, m_copyDataIntoNewContext);
        CopyContentIntoTempFlowNodeExecutionJobManager copyDataIntoTmpFlow = new CopyContentIntoTempFlowNodeExecutionJobManager(origResult);
        NodeExecutionJobManager oldJobManager = targetNode.getJobManager();
        tempWFM.setJobManager(targetNodeID, copyDataIntoTmpFlow);
        tempWFM.executeAllAndWaitUntilDoneInterruptibly();
        tempWFM.setJobManager(targetNodeID, oldJobManager);
        // do not use the cluster executor on the cluster...
        tempWFM.setJobManager(targetNodeID, NodeExecutionJobManagerPool.getDefaultJobManagerFactory().getInstance());
        if (!m_copyDataIntoNewContext) {
            copyFileStoreHandlerReference(targetNode, parent, false);
        }
        // save workflow in the local job dir
        if (m_localWorkflowDir != null) {
            tempWFM.save(m_localWorkflowDir, exec, true);
            deepCopyFilesInWorkflowDir(m_nc, tempWFM);
        }
        return new SandboxedNode(tempWFM, targetNodeID);
    } finally {
        portObjectRepositoryIDs.stream().forEach(PortObjectRepository::remove);
    }
}

Example 22 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class CopyContentIntoTempFlowNodeExecutionJob method mainExecute.

/**
 * {@inheritDoc}
 */
@Override
protected NodeContainerExecutionStatus mainExecute() {
    LoadResult lR = new LoadResult("load data into temp flow");
    getNodeContainer().loadExecutionResult(m_ncResult, new ExecutionMonitor(), lR);
    if (lR.hasErrors()) {
        LOGGER.error("Errors loading temporary data into workflow (to be submitted to cluster):\n" + lR.getFilteredError("", LoadResultEntryType.Warning));
    }
    return m_ncResult;
}

Example 23 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class AbstractSimplePortObject method equals.

/**
 * Method compares both <code>ModelContent</code> objects that first need
 * to be saved by calling {@link #save(ModelContentWO, ExecutionMonitor)}.
 * Override this method in order to compare both objects more efficiently.
 *
 * {@inheritDoc}
 */
@Override
public boolean equals(final Object oport) {
    if (oport == this) {
        return true;
    }
    if (oport == null) {
        return false;
    }
    if (!this.getClass().equals(oport.getClass())) {
        return false;
    }
    ModelContent tcont = new ModelContent("ignored");
    ModelContent ocont = new ModelContent("ignored");
    try {
        this.save(tcont, new ExecutionMonitor());
        ((AbstractSimplePortObject) oport).save(ocont, new ExecutionMonitor());
    } catch (CanceledExecutionException cee) {
    // ignored, should not happen
    }
    return tcont.equals(ocont);
}

Example 24 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class PMMLPreprocPortObjectView method create.

private void create() {
    // serialize port object
    synchronized (m_lock) {
        try {
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            TransformerHandler handler = PMMLPreprocPortObject.createTransformerHandlerForSave(out);
            for (PMMLPreprocOperation op : m_portObject.getOperations()) {
                String writeElement = op.getClass().getName();
                String locationElement = op.getTransformElement().toString();
                handler.startElement(null, null, writeElement, null);
                handler.startElement(null, null, locationElement, null);
                op.save(handler, new ExecutionMonitor());
                handler.endElement(null, null, locationElement);
                handler.endElement(null, null, writeElement);
                handler.endDocument();
            }
            out.close();
            SAXParserFactory saxFac = SAXParserFactory.newInstance();
            SAXParser parser = saxFac.newSAXParser();
            XMLTreeCreator treeCreator = new XMLTreeCreator();
            parser.parse(new InputSource(new ByteArrayInputStream(out.toByteArray())), treeCreator);
            m_tree.setModel(new DefaultTreeModel(treeCreator.getTreeNode()));
            add(new JScrollPane(m_tree));
            revalidate();
        } catch (Exception e) {
            // log and return a "error during saving" component
            LOGGER.error("PMML Preproc contains errors", e);
            PMMLPreprocPortObjectView.this.add(new JLabel("PMML Preproc contains errors: " + e.getMessage()));
        }
    }
}

Example 25 with ExecutionMonitor

use of org.knime.core.node.ExecutionMonitor in project knime-core by knime.

the class TreeDataCreator method readData.

/**
 * Reads the data from <b>learnData</b> into memory.
 * Each column is represented by a TreeColumnData object corresponding to its type
 * and whether it is a attribute or target column.
 *
 * @param learnData
 * @param configuration
 * @param exec
 * @return the TreeData object that holds all data in memory
 * @throws CanceledExecutionException
 */
public TreeData readData(final BufferedDataTable learnData, final TreeEnsembleLearnerConfiguration configuration, final ExecutionMonitor exec) throws CanceledExecutionException {
    if (learnData.size() <= 1) {
        throw new IllegalArgumentException("The input table must contain at least 2 rows!");
    }
    int index = 0;
    final long nrRows = learnData.size();
    final int nrLearnCols = m_attrColCreators.length;
    final boolean[] supportMissings = new boolean[nrLearnCols];
    for (int i = 0; i < nrLearnCols; i++) {
        supportMissings[i] = m_attrColCreators[i].acceptsMissing();
    }
    int rejectedMissings = 0;
    final int nrHilitePatterns = m_configuration.getNrHilitePatterns();
    // sort learnData according to the target column to enable equal size sampling
    final int targetColIdx = learnData.getDataTableSpec().findColumnIndex(m_configuration.getTargetColumn());
    Comparator<DataCell> targetComp = learnData.getDataTableSpec().getColumnSpec(targetColIdx).getType().getComparator();
    DataTableSorter sorter = new DataTableSorter(learnData, learnData.size(), new Comparator<DataRow>() {

        @Override
        public int compare(final DataRow arg0, final DataRow arg1) {
            return targetComp.compare(arg0.getCell(targetColIdx), arg1.getCell(targetColIdx));
        }
    });
    final ExecutionMonitor sortExec = exec.createSubProgress(0.5);
    final DataTable sortedTable = sorter.sort(sortExec);
    final ExecutionMonitor readExec = exec.createSubProgress(0.5);
    for (DataRow r : sortedTable) {
        double progress = index / (double) nrRows;
        readExec.setProgress(progress, "Row " + index + " of " + nrRows + " (\"" + r.getKey() + "\")");
        readExec.checkCanceled();
        boolean shouldReject = false;
        for (int i = 0; i < nrLearnCols; i++) {
            DataCell c = r.getCell(i);
            if (c.isMissing() && !supportMissings[i]) {
                shouldReject = true;
                break;
            }
        }
        DataCell targetCell = r.getCell(nrLearnCols);
        if (targetCell.isMissing()) {
            shouldReject = true;
        }
        if (shouldReject) {
            rejectedMissings += 1;
            continue;
        }
        if (index < nrHilitePatterns) {
            m_dataRowsForHiliteContainer.addRowToTable(r);
        }
        final RowKey key = r.getKey();
        for (int i = 0; i < nrLearnCols; i++) {
            DataCell c = r.getCell(i);
            m_attrColCreators[i].add(key, c);
        }
        m_targetColCreator.add(key, targetCell);
        index++;
    }
    if (nrHilitePatterns > 0 && index > nrHilitePatterns) {
        m_viewMessage = "Hilite (& color graphs) are based on a subset of " + "the data (" + nrHilitePatterns + "/" + index + ")";
    }
    if (rejectedMissings > 0) {
        StringBuffer warnMsgBuilder = new StringBuffer();
        warnMsgBuilder.append(rejectedMissings).append("/");
        warnMsgBuilder.append(learnData.size());
        warnMsgBuilder.append(" row(s) were ignored because they ");
        warnMsgBuilder.append("contain missing values.");
        m_warningMessage = warnMsgBuilder.toString();
    }
    CheckUtils.checkArgument(rejectedMissings < learnData.size(), "No rows left after removing missing values (table has %d row(s))", learnData.size());
    int nrLearnAttributes = 0;
    for (int i = 0; i < m_attrColCreators.length; i++) {
        nrLearnAttributes += m_attrColCreators[i].getNrAttributes();
    }
    TreeAttributeColumnData[] columns = new TreeAttributeColumnData[nrLearnAttributes];
    int learnAttributeIndex = 0;
    for (int i = 0; i < m_attrColCreators.length; i++) {
        TreeAttributeColumnDataCreator creator = m_attrColCreators[i];
        for (int a = 0; a < creator.getNrAttributes(); a++) {
            final TreeAttributeColumnData columnData = creator.createColumnData(a, configuration);
            columnData.getMetaData().setAttributeIndex(learnAttributeIndex);
            columns[learnAttributeIndex++] = columnData;
        }
    }
    TreeTargetColumnData targetCol = m_targetColCreator.createColumnData();
    return new TreeData(columns, targetCol, m_treeType);
}