use of org.knime.core.node.workflow.virtual.parchunk.ParallelizedChunkContent in project knime-core by knime.
the class WorkflowManager method parallelizeLoop.
/* Parallelize this "loop": create appropriate number of parallel
* branches executing the matching chunks.
*/
private void parallelizeLoop(final NodeID startID) throws IllegalLoopException {
try (WorkflowLock lock = lock()) {
final NodeID endID = m_workflow.getMatchingLoopEnd(startID);
LoopEndParallelizeNode endNode;
LoopStartParallelizeNode startNode;
try {
// just for validation
startNode = castNodeModel(startID, LoopStartParallelizeNode.class);
endNode = castNodeModel(endID, LoopEndParallelizeNode.class);
} catch (IllegalArgumentException iae) {
throw new IllegalLoopException("Parallel Chunk Start Node not connected to matching end node!", iae);
}
final ArrayList<NodeAndInports> loopBody = m_workflow.findAllNodesConnectedToLoopBody(startID, endID);
NodeID[] loopNodes = new NodeID[loopBody.size()];
loopNodes[0] = startID;
for (int i = 0; i < loopBody.size(); i++) {
loopNodes[i] = loopBody.get(i).getID();
}
// creating matching sub workflow node holding all chunks
Set<Pair<NodeID, Integer>> exposedInports = findNodesWithExternalSources(startID, loopNodes);
HashMap<Pair<NodeID, Integer>, Integer> extInConnections = new HashMap<Pair<NodeID, Integer>, Integer>();
PortType[] exposedInportTypes = new PortType[exposedInports.size() + 1];
// the first port is the variable port
exposedInportTypes[0] = FlowVariablePortObject.TYPE;
// the remaining ports cover the exposed inports of the loop body
int index = 1;
for (Pair<NodeID, Integer> npi : exposedInports) {
NodeContainer nc = getNodeContainer(npi.getFirst());
int portIndex = npi.getSecond();
exposedInportTypes[index] = nc.getInPort(portIndex).getPortType();
extInConnections.put(npi, index);
index++;
}
WorkflowManager subwfm = null;
if (startNode.getNrRemoteChunks() > 0) {
subwfm = createAndAddSubWorkflow(exposedInportTypes, new PortType[0], "Parallel Chunks");
NodeUIInformation startUIPlain = getNodeContainer(startID).getUIInformation();
if (startUIPlain != null) {
NodeUIInformation startUI = NodeUIInformation.builder(startUIPlain).translate(new int[] { 60, -60, 0, 0 }).build();
subwfm.setUIInformation(startUI);
}
// connect outside(!) nodes to new sub metanode
for (Map.Entry<Pair<NodeID, Integer>, Integer> entry : extInConnections.entrySet()) {
final Pair<NodeID, Integer> npi = entry.getKey();
int metanodeindex = entry.getValue();
if (metanodeindex >= 0) {
// ignore variable port!
// we need to find the source again (since our list
// only holds the destination...)
ConnectionContainer cc = this.getIncomingConnectionFor(npi.getFirst(), npi.getSecond());
this.addConnection(cc.getSource(), cc.getSourcePort(), subwfm.getID(), metanodeindex);
}
}
}
ParallelizedChunkContentMaster pccm = new ParallelizedChunkContentMaster(subwfm, endNode, startNode.getNrRemoteChunks());
for (int i = 0; i < startNode.getNrRemoteChunks(); i++) {
ParallelizedChunkContent copiedNodes = duplicateLoopBodyInSubWFMandAttach(subwfm, extInConnections, startID, endID, loopNodes, i);
copiedNodes.executeChunk();
pccm.addParallelChunk(i, copiedNodes);
}
// make sure head knows his chunk master (for potential cleanup)
startNode.setChunkMaster(pccm);
}
}
use of org.knime.core.node.workflow.virtual.parchunk.ParallelizedChunkContent in project knime-core by knime.
the class WorkflowManager method duplicateLoopBodyInSubWFMandAttach.
/*
* ...
* @param subWFM already prepared subworkflow with appropriate
* inports. If subWFM==this then the subworkflows are simply
* added to the same workflow.
* @param extInConnections map of incoming connections
* (NodeID + PortIndex) => WFM-Inport. Can be null if subWFM==this.
* ...
*/
private ParallelizedChunkContent duplicateLoopBodyInSubWFMandAttach(final WorkflowManager subWFM, final HashMap<Pair<NodeID, Integer>, Integer> extInConnections, final NodeID startID, final NodeID endID, final NodeID[] oldIDs, final int chunkIndex) {
assert m_workflowLock.isHeldByCurrentThread();
// compute offset for new nodes (shifted in case of same
// workflow, otherwise just underneath each other)
final int[] moveUIDist;
if (subWFM == this) {
moveUIDist = new int[] { (chunkIndex + 1) * 10, (chunkIndex + 1) * 80, 0, 0 };
} else {
moveUIDist = new int[] { (chunkIndex + 1) * 0, (chunkIndex + 1) * 150, 0, 0 };
}
// create virtual start node
NodeContainer startNode = getNodeContainer(startID);
// find port types (ignore Variable Port "ear")
PortType[] outTypes = new PortType[startNode.getNrOutPorts() - 1];
for (int i = 0; i < outTypes.length; i++) {
outTypes[i] = startNode.getOutPort(i + 1).getPortType();
}
NodeID virtualStartID = subWFM.createAndAddNode(new VirtualParallelizedChunkPortObjectInNodeFactory(outTypes));
NodeUIInformation startUIPlain = startNode.getUIInformation();
if (startUIPlain != null) {
NodeUIInformation startUI = NodeUIInformation.builder(startUIPlain).translate(moveUIDist).build();
subWFM.getNodeContainer(virtualStartID).setUIInformation(startUI);
}
// create virtual end node
NodeContainer endNode = getNodeContainer(endID);
assert endNode instanceof SingleNodeContainer;
// find port types (ignore Variable Port "ear")
PortType[] realInTypes = new PortType[endNode.getNrInPorts() - 1];
for (int i = 0; i < realInTypes.length; i++) {
realInTypes[i] = endNode.getInPort(i + 1).getPortType();
}
NodeID virtualEndID = subWFM.createAndAddNode(new VirtualParallelizedChunkPortObjectOutNodeFactory(realInTypes));
NodeUIInformation endUIPlain = endNode.getUIInformation();
if (endUIPlain != null) {
NodeUIInformation endUI = NodeUIInformation.builder(endUIPlain).translate(moveUIDist).build();
subWFM.getNodeContainer(virtualEndID).setUIInformation(endUI);
}
// copy nodes in loop body
WorkflowCopyContent.Builder copyContent = WorkflowCopyContent.builder();
copyContent.setNodeIDs(oldIDs);
WorkflowCopyContent newBody = subWFM.copyFromAndPasteHere(this, copyContent.build());
NodeID[] newIDs = newBody.getNodeIDs();
Map<NodeID, NodeID> oldIDsHash = new HashMap<NodeID, NodeID>();
for (int i = 0; i < oldIDs.length; i++) {
oldIDsHash.put(oldIDs[i], newIDs[i]);
NodeContainer nc = subWFM.getNodeContainer(newIDs[i]);
NodeUIInformation uiInfo = nc.getUIInformation();
if (uiInfo != null) {
nc.setUIInformation(NodeUIInformation.builder(uiInfo).translate(moveUIDist).build());
}
}
// restore connections to nodes outside the loop body (only incoming)
for (int i = 0; i < newIDs.length; i++) {
NodeContainer oldNode = getNodeContainer(oldIDs[i]);
for (int p = 0; p < oldNode.getNrInPorts(); p++) {
ConnectionContainer c = getIncomingConnectionFor(oldIDs[i], p);
if (c == null) {
// ignore: no incoming connection
} else if (oldIDsHash.containsKey(c.getSource())) {
// ignore: connection already retained by paste persistor
} else if (c.getSource().equals(startID)) {
// used to connect to start node, connect to virtual in now
subWFM.addConnection(virtualStartID, c.getSourcePort(), newIDs[i], c.getDestPort());
} else {
// source node not part of loop:
if (subWFM == this) {
addConnection(c.getSource(), c.getSourcePort(), newIDs[i], c.getDestPort());
} else {
// find new replacement port
int subWFMportIndex = extInConnections.get(new Pair<NodeID, Integer>(c.getDest(), c.getDestPort()));
subWFM.addConnection(subWFM.getID(), subWFMportIndex, newIDs[i], c.getDestPort());
}
}
}
}
// attach incoming connections of new Virtual End Node
for (int p = 0; p < endNode.getNrInPorts(); p++) {
ConnectionContainer c = getIncomingConnectionFor(endID, p);
if (c == null) {
// ignore: no incoming connection
} else if (oldIDsHash.containsKey(c.getSource())) {
// connects to node in loop - connect to copy
NodeID source = oldIDsHash.get(c.getSource());
subWFM.addConnection(source, c.getSourcePort(), virtualEndID, c.getDestPort());
} else if (c.getSource().equals(startID)) {
// used to connect to start node, connect to virtual in now
subWFM.addConnection(virtualStartID, c.getSourcePort(), virtualEndID, c.getDestPort());
} else {
// source node not part of loop
if (subWFM == this) {
addConnection(c.getSource(), c.getSourcePort(), virtualEndID, c.getDestPort());
} else {
// find new replacement port
int subWFMportIndex = extInConnections.get(new Pair<NodeID, Integer>(c.getSource(), c.getSourcePort()));
subWFM.addConnection(this.getID(), subWFMportIndex, virtualEndID, c.getDestPort());
}
}
}
if (subWFM == this) {
// connect start node var port with virtual start node
addConnection(startID, 0, virtualStartID, 0);
} else {
// add variable connection to port 0 of WFM!
if (this.canAddConnection(startID, 0, subWFM.getID(), 0)) {
// only add this one the first time...
this.addConnection(startID, 0, subWFM.getID(), 0);
}
subWFM.addConnection(subWFM.getID(), 0, virtualStartID, 0);
}
// set chunk of table to be processed in new virtual start node
LoopStartParallelizeNode startModel = castNodeModel(startID, LoopStartParallelizeNode.class);
VirtualParallelizedChunkNodeInput data = startModel.getVirtualNodeInput(chunkIndex);
VirtualParallelizedChunkPortObjectInNodeModel virtualInModel = subWFM.castNodeModel(virtualStartID, VirtualParallelizedChunkPortObjectInNodeModel.class);
virtualInModel.setVirtualNodeInput(data);
return new ParallelizedChunkContent(subWFM, virtualStartID, virtualEndID, newIDs);
}
Aggregations