use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.
the class ExecutionPlan method createIfAbsentReceiverTasklet.
private void createIfAbsentReceiverTasklet(EdgeDef edge, int[][] ptionsPerProcessor, int totalPtionCount) {
final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.get(edge.edgeId());
receiverMap.computeIfAbsent(edge.destVertex().vertexId(), x -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), x -> {
Map<Address, ReceiverTasklet> addrToTasklet = new HashMap<>();
// create a receiver per address
int offset = 0;
for (Address addr : ptionArrgmt.remotePartitionAssignment.get().keySet()) {
final OutboundCollector[] collectors = new OutboundCollector[ptionsPerProcessor.length];
// assign the queues starting from end
final int queueOffset = --offset;
Arrays.setAll(collectors, n -> new ConveyorCollector(localConveyors[n], localConveyors[n].queueCount() + queueOffset, ptionsPerProcessor[n]));
final OutboundCollector collector = compositeCollector(collectors, edge, totalPtionCount);
ReceiverTasklet receiverTasklet = new ReceiverTasklet(collector, edge.getConfig().getReceiveWindowMultiplier(), getConfig().getInstanceConfig().getFlowControlPeriodMs());
addrToTasklet.put(addr, receiverTasklet);
}
return addrToTasklet;
});
}
use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.
the class ExecutionPlan method createOutboundCollectors.
private OutboundCollector[] createOutboundCollectors(EdgeDef edge, int processorIndex, Map<Address, ConcurrentConveyor<Object>> senderConveyorMap) {
final int upstreamParallelism = edge.sourceVertex().localParallelism();
final int downstreamParallelism = edge.destVertex().localParallelism();
final int numRemoteMembers = ptionArrgmt.remotePartitionAssignment.get().size();
final int queueSize = edge.getConfig().getQueueSize();
final int[][] ptionsPerProcessor = ptionArrgmt.assignPartitionsToProcessors(downstreamParallelism, edge.isDistributed());
if (edge.routingPolicy() == RoutingPolicy.ISOLATED) {
if (downstreamParallelism < upstreamParallelism) {
throw new IllegalArgumentException(String.format("The edge %s specifies the %s routing policy, but the downstream vertex" + " parallelism (%d) is less than the upstream vertex parallelism (%d)", edge, RoutingPolicy.ISOLATED.name(), downstreamParallelism, upstreamParallelism));
}
if (edge.isDistributed()) {
throw new IllegalArgumentException("Isolated edges must be local: " + edge);
}
// there is only one producer per consumer for a one to many edge, so queueCount is always 1
ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> createConveyorArray(downstreamParallelism, 1, queueSize));
return IntStream.range(0, downstreamParallelism).filter(i -> i % upstreamParallelism == processorIndex).mapToObj(i -> new ConveyorCollector(localConveyors[i], 0, ptionsPerProcessor[i])).toArray(OutboundCollector[]::new);
}
/*
* Each edge is represented by an array of conveyors between the producers and consumers
* There are as many conveyors as there are consumers.
* Each conveyor has one queue per producer.
*
* For a distributed edge, there is one additional producer per member represented
* by the ReceiverTasklet.
*/
final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> {
int queueCount = upstreamParallelism + (edge.isDistributed() ? numRemoteMembers : 0);
return createConveyorArray(downstreamParallelism, queueCount, queueSize);
});
final OutboundCollector[] localCollectors = new OutboundCollector[downstreamParallelism];
Arrays.setAll(localCollectors, n -> new ConveyorCollector(localConveyors[n], processorIndex, ptionsPerProcessor[n]));
// in a local edge, we only have the local collectors.
if (!edge.isDistributed()) {
return localCollectors;
}
// in a distributed edge, allCollectors[0] is the composite of local collectors, and
// allCollectors[n] where n > 0 is a collector pointing to a remote member _n_.
final int totalPtionCount = nodeEngine.getPartitionService().getPartitionCount();
final OutboundCollector[] allCollectors;
createIfAbsentReceiverTasklet(edge, ptionsPerProcessor, totalPtionCount);
// assign remote partitions to outbound data collectors
final Map<Address, int[]> memberToPartitions = ptionArrgmt.remotePartitionAssignment.get();
allCollectors = new OutboundCollector[memberToPartitions.size() + 1];
allCollectors[0] = compositeCollector(localCollectors, edge, totalPtionCount);
int index = 1;
for (Map.Entry<Address, int[]> entry : memberToPartitions.entrySet()) {
allCollectors[index++] = new ConveyorCollectorWithPartition(senderConveyorMap.get(entry.getKey()), processorIndex, entry.getValue());
}
return allCollectors;
}
use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast by hazelcast.
the class Networking method createFlowControlPacket.
private Map<Address, byte[]> createFlowControlPacket() throws IOException {
class MemberData {
final BufferObjectDataOutput output = createObjectDataOutput(nodeEngine, lastFlowPacketSize);
final Connection memberConnection;
Long startedExecutionId;
MemberData(Address address) {
memberConnection = getMemberConnection(nodeEngine, address);
}
}
Map<Address, MemberData> res = new HashMap<>();
for (ExecutionContext execCtx : jobExecutionService.getExecutionContexts()) {
Map<SenderReceiverKey, ReceiverTasklet> receiverMap = execCtx.receiverMap();
if (receiverMap == null) {
continue;
}
for (Entry<SenderReceiverKey, ReceiverTasklet> en : receiverMap.entrySet()) {
assert !en.getKey().address.equals(nodeEngine.getThisAddress());
MemberData md = res.computeIfAbsent(en.getKey().address, address -> new MemberData(address));
if (md.startedExecutionId == null) {
md.startedExecutionId = execCtx.executionId();
md.output.writeLong(md.startedExecutionId);
}
assert en.getKey().vertexId != TERMINAL_VERTEX_ID;
md.output.writeInt(en.getKey().vertexId);
md.output.writeInt(en.getKey().ordinal);
md.output.writeInt(en.getValue().updateAndGetSendSeqLimitCompressed(md.memberConnection));
}
for (MemberData md : res.values()) {
if (md.startedExecutionId != null) {
// write a mark to terminate values for an execution
md.output.writeInt(TERMINAL_VERTEX_ID);
md.startedExecutionId = null;
}
}
}
for (MemberData md : res.values()) {
assert md.output.position() > 0;
// write a mark to terminate all executions
// Execution IDs are generated using Flake ID generator and those are >0 normally, we
// use MIN_VALUE as a terminator.
md.output.writeLong(TERMINAL_EXECUTION_ID);
}
// finalize the packets
int maxSize = 0;
for (Entry<Address, MemberData> entry : res.entrySet()) {
byte[] data = entry.getValue().output.toByteArray();
// we break type safety to avoid creating a new map, we replace the values to a different type in place
@SuppressWarnings({ "unchecked", "rawtypes" }) Entry<Address, byte[]> entry1 = (Entry) entry;
entry1.setValue(data);
if (data.length > maxSize) {
maxSize = data.length;
}
}
lastFlowPacketSize = maxSize;
return (Map) res;
}
use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast by hazelcast.
the class ExecutionPlan method initialize.
/**
* A method called on the members as part of the InitExecutionOperation.
* Creates tasklets, inboxes/outboxes and connects these to make them ready
* for a later StartExecutionOperation.
*/
public void initialize(NodeEngineImpl nodeEngine, long jobId, long executionId, @Nonnull SnapshotContext snapshotContext, ConcurrentHashMap<String, File> tempDirectories, InternalSerializationService jobSerializationService) {
this.nodeEngine = nodeEngine;
this.jobClassLoaderService = ((JetServiceBackend) nodeEngine.getService(JetServiceBackend.SERVICE_NAME)).getJobClassLoaderService();
this.executionId = executionId;
initProcSuppliers(jobId, tempDirectories, jobSerializationService);
initDag(jobSerializationService);
this.ptionArrgmt = new PartitionArrangement(partitionAssignment, nodeEngine.getThisAddress());
Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(vertices);
for (Address destAddr : remoteMembers.get()) {
Connection conn = getMemberConnection(nodeEngine, destAddr);
if (conn == null) {
throw new TopologyChangedException("no connection to job participant: " + destAddr);
}
memberConnections.put(destAddr, conn);
}
for (VertexDef vertex : vertices) {
ClassLoader processorClassLoader = isLightJob ? null : jobClassLoaderService.getProcessorClassLoader(jobId, vertex.name());
Collection<? extends Processor> processors = doWithClassLoader(processorClassLoader, () -> createProcessors(vertex, vertex.localParallelism()));
String jobPrefix = prefix(jobConfig.getName(), jobId, vertex.name());
// create StoreSnapshotTasklet and the queues to it
ConcurrentConveyor<Object> ssConveyor = null;
if (!isLightJob) {
// Note that we create the snapshot queues for all non-light jobs, even if they don't have
// processing guarantee enabled, because in EE one can request a snapshot also for
// non-snapshotted jobs.
@SuppressWarnings("unchecked") QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
ILogger storeSnapshotLogger = prefixedLogger(nodeEngine.getLogger(StoreSnapshotTasklet.class), jobPrefix);
StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, ConcurrentInboundEdgeStream.create(ssConveyor, 0, 0, true, jobPrefix + "/ssFrom", null), new AsyncSnapshotWriterImpl(nodeEngine, snapshotContext, vertex.name(), memberIndex, memberCount, jobSerializationService), storeSnapshotLogger, vertex.name(), higherPriorityVertices.contains(vertex.vertexId()));
tasklets.add(ssTasklet);
}
int localProcessorIdx = 0;
for (Processor processor : processors) {
int globalProcessorIndex = memberIndex * vertex.localParallelism() + localProcessorIdx;
String processorPrefix = prefix(jobConfig.getName(), jobId, vertex.name(), globalProcessorIndex);
ILogger logger = prefixedLogger(nodeEngine.getLogger(processor.getClass()), processorPrefix);
ProcCtx context = new ProcCtx(nodeEngine, jobId, executionId, getJobConfig(), logger, vertex.name(), localProcessorIdx, globalProcessorIndex, isLightJob, partitionAssignment, vertex.localParallelism(), memberIndex, memberCount, tempDirectories, jobSerializationService, subject, processorClassLoader);
// createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
// Also populates instance fields: senderMap, receiverMap, tasklets.
List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, jobSerializationService);
List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, globalProcessorIndex);
OutboundCollector snapshotCollector = ssConveyor == null ? null : new ConveyorCollector(ssConveyor, localProcessorIdx, null);
// vertices which are only used for snapshot restore will not be marked as "source=true" in metrics
// also do not consider snapshot restore edges for determining source tag
boolean isSource = vertex.inboundEdges().stream().allMatch(EdgeDef::isSnapshotRestoreEdge) && !vertex.isSnapshotVertex();
ProcessorTasklet processorTasklet = new ProcessorTasklet(context, nodeEngine.getExecutionService().getExecutor(TASKLET_INIT_CLOSE_EXECUTOR_NAME), jobSerializationService, processor, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, isSource);
tasklets.add(processorTasklet);
this.processors.add(processor);
localProcessorIdx++;
}
}
List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
tasklets.addAll(allReceivers);
}
use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.
the class ExecutionPlan method initialize.
public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
this.nodeEngine = nodeEngine;
this.executionId = executionId;
initProcSuppliers();
initDag();
this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
JetInstance instance = getJetInstance(nodeEngine);
for (VertexDef vertex : vertices) {
Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
// create StoreSnapshotTasklet and the queues to it
QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
tasklets.add(ssTasklet);
int localProcessorIdx = 0;
for (Processor p : processors) {
int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
// createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
// Also populates instance fields: senderMap, receiverMap, tasklets.
List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
tasklets.add(processorTasklet);
this.processors.add(p);
localProcessorIdx++;
}
}
List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
tasklets.addAll(allReceivers);
}
Aggregations