Examples with ConcurrentConveyor com.hazelcast.internal.util.concurrent.ConcurrentConveyor used on opensource projects

Example 6 with ConcurrentConveyor

use of com.hazelcast.internal.util.concurrent.ConcurrentConveyor in project hazelcast by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge, String jobPrefix, InternalSerializationService jobSerializationService) {
    assert !edge.isLocal() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            @SuppressWarnings("unchecked") ComparatorEx<Object> origComparator = (ComparatorEx<Object>) edge.getOrderComparator();
            ComparatorEx<ObjectWithPartitionId> adaptedComparator = origComparator == null ? null : (l, r) -> origComparator.compare(l.getItem(), r.getItem());
            final InboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, jobPrefix + "/toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr, adaptedComparator);
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, memberConnections.get(destAddr), destVertexId, edge.getConfig().getPacketSizeLimit(), executionId, edge.sourceVertex().name(), edge.sourceOrdinal(), jobSerializationService);
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}

Example 7 with ConcurrentConveyor

use of com.hazelcast.internal.util.concurrent.ConcurrentConveyor in project hazelcast by hazelcast.

the class ExecutionPlan method createIfAbsentReceiverTasklet.

private void createIfAbsentReceiverTasklet(EdgeDef edge, String jobPrefix, int[][] ptionsPerProcessor, int totalPtionCount, InternalSerializationService jobSerializationService) {
    final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.get(edge.edgeId());
    receiverMap.computeIfAbsent(edge.destVertex().vertexId(), x -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), x -> {
        Map<Address, ReceiverTasklet> addrToTasklet = new HashMap<>();
        // create a receiver per address
        int offset = 0;
        for (Address addr : ptionArrgmt.getRemotePartitionAssignment().keySet()) {
            final OutboundCollector[] collectors = new OutboundCollector[ptionsPerProcessor.length];
            // assign the queues starting from end
            final int queueOffset = --offset;
            Arrays.setAll(collectors, n -> new ConveyorCollector(localConveyors[n], localConveyors[n].queueCount() + queueOffset, ptionsPerProcessor[n]));
            final OutboundCollector collector = compositeCollector(collectors, edge, totalPtionCount, true);
            ReceiverTasklet receiverTasklet = new ReceiverTasklet(collector, jobSerializationService, edge.getConfig().getReceiveWindowMultiplier(), getJetConfig().getFlowControlPeriodMs(), nodeEngine.getLoggingService(), addr, edge.destOrdinal(), edge.destVertex().name(), memberConnections.get(addr), jobPrefix);
            addrToTasklet.put(addr, receiverTasklet);
        }
        return addrToTasklet;
    });
}

Example 8 with ConcurrentConveyor

use of com.hazelcast.internal.util.concurrent.ConcurrentConveyor in project hazelcast-jet by hazelcast.

the class ExecutionPlan method initialize.

public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
    this.nodeEngine = nodeEngine;
    this.executionId = executionId;
    initProcSuppliers();
    initDag();
    this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
    JetInstance instance = getJetInstance(nodeEngine);
    for (VertexDef vertex : vertices) {
        Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
        // create StoreSnapshotTasklet and the queues to it
        QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
        Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
        ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
        StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
        tasklets.add(ssTasklet);
        int localProcessorIdx = 0;
        for (Processor p : processors) {
            int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
            String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
            ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
            String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
            ((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
            OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
            tasklets.add(processorTasklet);
            this.processors.add(p);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}

Example 9 with ConcurrentConveyor

use of com.hazelcast.internal.util.concurrent.ConcurrentConveyor in project hazelcast-jet by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge) {
    assert edge.isDistributed() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            final ConcurrentInboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, "sender-toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr.toString().replace('.', '-'));
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, executionId, destVertexId, edge.getConfig().getPacketSizeLimit());
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}