Search in sources :

Example 1 with ConcurrentInboundEdgeStream

use of com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream in project hazelcast-jet by hazelcast.

the class ExecutionPlan method initialize.

public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
    this.nodeEngine = nodeEngine;
    this.executionId = executionId;
    initProcSuppliers();
    initDag();
    this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
    JetInstance instance = getJetInstance(nodeEngine);
    for (VertexDef vertex : vertices) {
        Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
        // create StoreSnapshotTasklet and the queues to it
        QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
        Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
        ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
        StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
        tasklets.add(ssTasklet);
        int localProcessorIdx = 0;
        for (Processor p : processors) {
            int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
            String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
            ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
            String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
            ((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
            OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
            tasklets.add(processorTasklet);
            this.processors.add(p);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}
Also used : Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.nio.Address) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) Util.writeList(com.hazelcast.jet.impl.util.Util.writeList) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Util.readList(com.hazelcast.jet.impl.util.Util.readList) List(java.util.List) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) IntStream(java.util.stream.IntStream) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) IPartitionService(com.hazelcast.spi.partition.IPartitionService) IOException(java.io.IOException) NodeEngine(com.hazelcast.spi.NodeEngine) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetService(com.hazelcast.jet.impl.JetService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Processor(com.hazelcast.jet.core.Processor) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe)

Example 2 with ConcurrentInboundEdgeStream

use of com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream in project hazelcast-jet by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge) {
    assert edge.isDistributed() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            final ConcurrentInboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, "sender-toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr.toString().replace('.', '-'));
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, executionId, destVertexId, edge.getConfig().getPacketSizeLimit());
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}
Also used : Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.nio.Address) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) Util.writeList(com.hazelcast.jet.impl.util.Util.writeList) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Util.readList(com.hazelcast.jet.impl.util.Util.readList) List(java.util.List) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) IntStream(java.util.stream.IntStream) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) IPartitionService(com.hazelcast.spi.partition.IPartitionService) IOException(java.io.IOException) NodeEngine(com.hazelcast.spi.NodeEngine) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetService(com.hazelcast.jet.impl.JetService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Address(com.hazelcast.nio.Address) HashMap(java.util.HashMap) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream)

Aggregations

ConcurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor)2 ConcurrentConveyor.concurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor)2 OneToOneConcurrentArrayQueue (com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue)2 QueuedPipe (com.hazelcast.internal.util.concurrent.QueuedPipe)2 JetException (com.hazelcast.jet.JetException)2 JetInstance (com.hazelcast.jet.JetInstance)2 DEFAULT_QUEUE_SIZE (com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE)2 JetConfig (com.hazelcast.jet.config.JetConfig)2 JobConfig (com.hazelcast.jet.config.JobConfig)2 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)2 RoutingPolicy (com.hazelcast.jet.core.Edge.RoutingPolicy)2 Processor (com.hazelcast.jet.core.Processor)2 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)2 JetService (com.hazelcast.jet.impl.JetService)2 ConcurrentInboundEdgeStream (com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream)2 ConveyorCollector (com.hazelcast.jet.impl.execution.ConveyorCollector)2 ConveyorCollectorWithPartition (com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition)2 InboundEdgeStream (com.hazelcast.jet.impl.execution.InboundEdgeStream)2 OutboundCollector (com.hazelcast.jet.impl.execution.OutboundCollector)2 OutboundCollector.compositeCollector (com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector)2