use of com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream in project hazelcast-jet by hazelcast.
the class ExecutionPlan method initialize.
public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
this.nodeEngine = nodeEngine;
this.executionId = executionId;
initProcSuppliers();
initDag();
this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
JetInstance instance = getJetInstance(nodeEngine);
for (VertexDef vertex : vertices) {
Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
// create StoreSnapshotTasklet and the queues to it
QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
tasklets.add(ssTasklet);
int localProcessorIdx = 0;
for (Processor p : processors) {
int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
// createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
// Also populates instance fields: senderMap, receiverMap, tasklets.
List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
tasklets.add(processorTasklet);
this.processors.add(p);
localProcessorIdx++;
}
}
List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
tasklets.addAll(allReceivers);
}
use of com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream in project hazelcast-jet by hazelcast.
the class ExecutionPlan method memberToSenderConveyorMap.
/**
* Creates (if absent) for the given edge one sender tasklet per remote member,
* each with a single conveyor with a number of producer queues feeding it.
* Populates the {@link #senderMap} and {@link #tasklets} fields.
*/
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge) {
assert edge.isDistributed() : "Edge is not distributed";
return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
for (Address destAddr : remoteMembers.get()) {
final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
final ConcurrentInboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, "sender-toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr.toString().replace('.', '-'));
final int destVertexId = edge.destVertex().vertexId();
final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, executionId, destVertexId, edge.getConfig().getPacketSizeLimit());
senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
tasklets.add(t);
addrToConveyor.put(destAddr, conveyor);
}
return addrToConveyor;
});
}
Aggregations