Examples with PortObjectIDSettings org.knime.core.node.exec.dataexchange.PortObjectIDSettings used on opensource projects

Example 1 with PortObjectIDSettings

use of org.knime.core.node.exec.dataexchange.PortObjectIDSettings in project knime-core by knime.

the class SandboxedNodeCreator method createSandbox.

/**
 * Creates that temporary mini workflow that is executed remotely on the cluster/stream executor.
 * The returned value should be {@link SandboxedNode#close()} when done (using try-with-resources). After this
 * method is called no other set-method should be called.
 *
 * @param exec for progress/cancelation
 * @return the index of the node that represents this node (the node to execute) in the temporary mini workflow
 * @throws InvalidSettingsException
 * @throws IOException
 * @throws CanceledExecutionException
 * @throws LockFailedException
 * @throws InterruptedException
 */
public SandboxedNode createSandbox(final ExecutionMonitor exec) throws InvalidSettingsException, IOException, CanceledExecutionException, LockFailedException, InterruptedException {
    exec.setMessage("Creating virtual workflow");
    final WorkflowManager parent = m_nc.getParent();
    // derive workflow context via NodeContext as the parent could only a be a metanode in a metanode...
    final WorkflowContext origContext = NodeContext.getContext().getWorkflowManager().getContext();
    WorkflowContext.Factory ctxFactory;
    // (specifically reading knime://knime.workflow files)
    if (!m_copyDataIntoNewContext) {
        ctxFactory = origContext.createCopy();
        if (m_localWorkflowDir != null) {
            ctxFactory.setOriginalLocation(origContext.getCurrentLocation()).setCurrentLocation(m_localWorkflowDir);
        }
    } else if (m_localWorkflowDir != null) {
        ctxFactory = new WorkflowContext.Factory(m_localWorkflowDir);
    } else {
        ctxFactory = new WorkflowContext.Factory(FileUtil.createTempDir("sandbox-" + m_nc.getNameWithID()));
    }
    // We have to use the same location for the temporary files
    ctxFactory.setTempLocation(origContext.getTempLocation());
    origContext.getMountpointURI().ifPresent(u -> ctxFactory.setMountpointURI(u));
    WorkflowCreationHelper creationHelper = new WorkflowCreationHelper();
    creationHelper.setWorkflowContext(ctxFactory.createContext());
    if (!m_copyDataIntoNewContext) {
        creationHelper.setWorkflowDataRepository(parent.getWorkflowDataRepository());
    }
    WorkflowManager tempWFM = m_rootWFM.createAndAddProject("Sandbox Exec on " + m_nc.getNameWithID(), creationHelper);
    // Add the workflow variables
    List<FlowVariable> workflowVariables = parent.getProjectWFM().getWorkflowVariables();
    tempWFM.addWorkflowVariables(true, workflowVariables.toArray(new FlowVariable[workflowVariables.size()]));
    // update credentials store of the workflow
    CredentialsStore cs = tempWFM.getCredentialsStore();
    workflowVariables.stream().filter(f -> f.getType().equals(FlowVariable.Type.CREDENTIALS)).filter(f -> !cs.contains(f.getName())).forEach(cs::addFromFlowVariable);
    final int inCnt = m_inData.length;
    // port object IDs in static port object map, one entry for
    // each connected input (no value for unconnected optional inputs)
    List<UUID> portObjectRepositoryIDs = new ArrayList<>(inCnt);
    try {
        NodeID[] ins = new NodeID[inCnt];
        for (int i = 0; i < inCnt; i++) {
            final PortObject in = m_inData[i];
            final NodeInPort inPort = m_nc.getInPort(i);
            final PortType portType = inPort.getPortType();
            if (in == null) {
                // unconnected optional input
                CheckUtils.checkState(portType.isOptional(), "No data at port %d, although port is mandatory (port type %s)", i, portType.getName());
                continue;
            }
            List<FlowVariable> flowVars = getFlowVariablesOnPort(i);
            UUID portObjectRepositoryID = PortObjectRepository.add(in);
            PortObjectIDSettings settings = new PortObjectIDSettings();
            settings.setId(portObjectRepositoryID);
            settings.setCopyData(m_copyDataIntoNewContext);
            settings.setFlowVariables(flowVars);
            portObjectRepositoryIDs.add(portObjectRepositoryID);
            boolean isTable = BufferedDataTable.TYPE.equals(portType);
            NodeID inID = tempWFM.createAndAddNode(isTable ? TABLE_READ_NODE_FACTORY : OBJECT_READ_NODE_FACTORY);
            NodeSettings s = new NodeSettings("temp_data_in");
            tempWFM.saveNodeSettings(inID, s);
            PortObjectInNodeModel.setInputNodeSettings(s, settings);
            // update credentials store of the workflow
            flowVars.stream().filter(f -> f.getType().equals(FlowVariable.Type.CREDENTIALS)).filter(f -> !cs.contains(f.getName())).forEach(cs::addFromFlowVariable);
            tempWFM.loadNodeSettings(inID, s);
            ins[i] = inID;
        }
        // execute inPort object nodes to store the input data in them
        if (ins.length > 0 && !tempWFM.executeAllAndWaitUntilDoneInterruptibly()) {
            String error = "Unable to execute virtual workflow, status sent to log facilities";
            LOGGER.debug(error + ":");
            LOGGER.debug(tempWFM.toString());
            throw new RuntimeException(error);
        }
        // add the target node to the workflow
        WorkflowCopyContent.Builder content = WorkflowCopyContent.builder();
        content.setNodeIDs(m_nc.getID());
        final NodeID targetNodeID = tempWFM.copyFromAndPasteHere(parent, content.build()).getNodeIDs()[0];
        NodeContainer targetNode = tempWFM.getNodeContainer(targetNodeID);
        // connect target node to inPort object nodes, skipping unconnected (optional) inputs
        IntStream.range(0, inCnt).filter(i -> ins[i] != null).forEach(i -> tempWFM.addConnection(ins[i], 1, targetNodeID, i));
        if (m_forwardConnectionProgressEvents) {
            setupConnectionProgressEventListeners(m_nc, targetNode);
        }
        // copy the existing tables into the (meta) node (e.g. an executed file reader that's necessary
        // for other nodes to execute)
        exec.setMessage("Copying tables into temp flow");
        NodeContainerExecutionResult origResult = m_nc.createExecutionResult(exec);
        ExecutionMonitor copyExec = exec.createSubProgress(0.0);
        copyExistingTablesIntoSandboxContainer(origResult, m_nc, targetNode, copyExec, m_copyDataIntoNewContext);
        CopyContentIntoTempFlowNodeExecutionJobManager copyDataIntoTmpFlow = new CopyContentIntoTempFlowNodeExecutionJobManager(origResult);
        NodeExecutionJobManager oldJobManager = targetNode.getJobManager();
        tempWFM.setJobManager(targetNodeID, copyDataIntoTmpFlow);
        tempWFM.executeAllAndWaitUntilDoneInterruptibly();
        tempWFM.setJobManager(targetNodeID, oldJobManager);
        // do not use the cluster executor on the cluster...
        tempWFM.setJobManager(targetNodeID, NodeExecutionJobManagerPool.getDefaultJobManagerFactory().getInstance());
        if (!m_copyDataIntoNewContext) {
            copyFileStoreHandlerReference(targetNode, parent, false);
        }
        // save workflow in the local job dir
        if (m_localWorkflowDir != null) {
            tempWFM.save(m_localWorkflowDir, exec, true);
            deepCopyFilesInWorkflowDir(m_nc, tempWFM);
        }
        return new SandboxedNode(tempWFM, targetNodeID);
    } finally {
        portObjectRepositoryIDs.stream().forEach(PortObjectRepository::remove);
    }
}

Example 2 with PortObjectIDSettings

use of org.knime.core.node.exec.dataexchange.PortObjectIDSettings in project knime-core by knime.

the class PortObjectInNodeModel method getInputNodeSettingsCopy.

/**
 * @return a copy of the settings representing how the provided port object is referenced
 */
public PortObjectIDSettings getInputNodeSettingsCopy() {
    final NodeSettings settings = new NodeSettings("copy");
    m_portObjectIDSettings.saveSettings(settings);
    final PortObjectIDSettings copy = new PortObjectIDSettings();
    try {
        copy.loadSettings(settings);
    } catch (InvalidSettingsException ex) {
        throw new RuntimeException(ex);
    }
    return copy;
}

Example 3 with PortObjectIDSettings

use of org.knime.core.node.exec.dataexchange.PortObjectIDSettings in project knime-core by knime.

the class ReferenceReaderDataUtil method writeReferenceReaderData.

/**
 * Writes the 'reference reader node'-data of a workflow to file.
 *
 * @param wfm a {@link WorkflowManager}
 * @param portObjectReaderSufIds a set of the referenced reader node identifiers
 * @param tmpDataDir the data directory
 * @param exec a {@link ExecutionMonitor}
 * @throws IOException
 * @throws CanceledExecutionException
 * @throws URISyntaxException
 * @throws InvalidSettingsException
 */
public static void writeReferenceReaderData(final WorkflowManager wfm, final Set<NodeIDSuffix> portObjectReaderSufIds, final File tmpDataDir, final ExecutionMonitor exec) throws IOException, CanceledExecutionException, URISyntaxException, InvalidSettingsException {
    // reconfigure reference reader nodes and store their data in temp directory
    exec.setMessage(() -> "Introducing reference reader nodes.");
    for (NodeIDSuffix portObjectReaderSufId : portObjectReaderSufIds) {
        final NodeID portObjectReaderId = portObjectReaderSufId.prependParent(wfm.getID());
        final var portObjectReaderNC = wfm.findNodeContainer(portObjectReaderId);
        assert portObjectReaderNC instanceof NativeNodeContainer;
        final var portObjectReaderNM = ((NativeNodeContainer) portObjectReaderNC).getNodeModel();
        assert portObjectReaderNM instanceof PortObjectInNodeModel;
        final PortObjectInNodeModel portObjectReader = (PortObjectInNodeModel) portObjectReaderNM;
        final Optional<PortObject> poOpt = portObjectReader.getPortObject();
        assert poOpt.isPresent();
        PortObject po = poOpt.get();
        if (po instanceof WorkflowPortObject) {
            // also write the data for potential reference reader nodes within a referenced workflow segment
            // AP-16062
            po = writeReferenceReaderDataForWorkflowPort((WorkflowPortObject) po, tmpDataDir, exec);
        }
        final var poFileName = portObjectReaderSufId.toString().replace(":", "_") + "_" + System.identityHashCode(po);
        final var poFileRelativeURI = new URI("knime://knime.workflow/data/" + poFileName);
        final var tmpPoFile = new File(tmpDataDir, poFileName);
        final PortObjectIDSettings poSettings = portObjectReader.getInputNodeSettingsCopy();
        if (po instanceof BufferedDataTable) {
            final BufferedDataTable table = (BufferedDataTable) po;
            DataContainer.writeToZip(table, tmpPoFile, exec.createSubProgress(.2 / portObjectReaderSufIds.size()));
            poSettings.setFileReference(poFileRelativeURI, true);
        } else {
            PortUtil.writeObjectToFile(po, tmpPoFile, exec.createSubProgress(.2 / portObjectReaderSufIds.size()));
            poSettings.setFileReference(poFileRelativeURI, false);
        }
        final var settings = new NodeSettings("root");
        portObjectReaderNC.getParent().saveNodeSettings(portObjectReaderId, settings);
        final NodeSettingsWO modelSettings = settings.addNodeSettings("model");
        poSettings.saveSettings(modelSettings);
        portObjectReaderNC.getParent().loadNodeSettings(portObjectReaderId, settings);
    }
}

Example 4 with PortObjectIDSettings

use of org.knime.core.node.exec.dataexchange.PortObjectIDSettings in project knime-core by knime.

the class PortObjectInNodeModel method setInputNodeSettings.

/**
 * Injects the port object ID and flow variables into the settings of
 * this node. If this node reads back the settings, it will read the port
 * object with the specified ID and variables (during execute)
 *
 * @param s settings object with settings of this node.
 * @param portObjectID the ID of the port object to inject
 * @param flowVariables The flow variables the node should expose
 * @param copyData Wether to deep-clone data (context switch)
 * @throws InvalidSettingsException if the settings are invalid.
 */
public static void setInputNodeSettings(final NodeSettings s, final int portObjectID, final List<FlowVariable> flowVariables, final boolean copyData) throws InvalidSettingsException {
    PortObjectIDSettings poSettings = new PortObjectIDSettings();
    poSettings.setId(portObjectID);
    poSettings.setCopyData(copyData);
    poSettings.setFlowVariables(flowVariables);
    if (!s.containsKey("model")) {
        s.addNodeSettings("model");
    }
    NodeSettings modelSettings = s.getNodeSettings("model");
    poSettings.saveSettings(modelSettings);
}

Example 5 with PortObjectIDSettings

use of org.knime.core.node.exec.dataexchange.PortObjectIDSettings in project knime-core by knime.

the class ReferenceReaderDataUtil method copyReferenceReaderData.

/**
 * Copies the 'reference reader node'-data to the {@link PortObjectRepository}, returns the corresponding nodes id.
 *
 * Note: this operation potentially manipulates the passed workflow manager (changing the port object reference
 * type, e.g. from 'file' to 'repository')
 *
 * @param wfm <T> The specific PortObject class of interest.
 * @param exec a {@link ExecutionContext}
 * @param portObjRepoNodeModel special node model that receives the port objects and makes them available through the
 *            {@link PortObjectRepository}
 * @return a set of {@link NodeIDSuffix}.
 * @throws IOException
 * @throws CanceledExecutionException
 * @throws InvalidSettingsException
 */
public static Set<NodeIDSuffix> copyReferenceReaderData(final WorkflowManager wfm, final ExecutionContext exec, final AbstractPortObjectRepositoryNodeModel portObjRepoNodeModel) throws InvalidSettingsException, CanceledExecutionException, IOException {
    Set<NodeIDSuffix> res = new HashSet<>();
    for (var nc : wfm.getNodeContainers()) {
        if (nc instanceof NativeNodeContainer && ((NativeNodeContainer) nc).getNodeModel() instanceof PortObjectInNodeModel) {
            exec.setProgress("Copying data for node " + nc.getID());
            PortObjectInNodeModel portObjectReader = (PortObjectInNodeModel) ((NativeNodeContainer) nc).getNodeModel();
            final PortObjectIDSettings poSettings = portObjectReader.getInputNodeSettingsCopy();
            if (poSettings.getReferenceType() != ReferenceType.FILE) {
                throw new IllegalStateException("Reference reader nodes expected to reference a file. But the reference type is " + poSettings.getReferenceType());
            }
            var uri = poSettings.getUri();
            var wfFile = wfm.getNodeContainerDirectory().getFile();
            var absoluteDataFile = new File(wfFile, uri.toString().replace("knime://knime.workflow", ""));
            if (!absoluteDataFile.getCanonicalPath().startsWith(wfFile.getCanonicalPath())) {
                throw new IllegalStateException("Trying to read in a data file outside of the workflow directory. Not allowed!");
            }
            var po = readPortObjectFromFile(absoluteDataFile, exec, poSettings.isTable());
            var uuid = UUID.randomUUID();
            portObjRepoNodeModel.addPortObject(uuid, po);
            PortObjectRepository.add(uuid, po);
            updatePortObjectReferenceReaderReference(wfm, nc.getID(), poSettings, uuid);
            res.add(NodeIDSuffix.create(wfm.getID(), nc.getID()));
        }
    }
    return res;
}