Search in sources :

Example 1 with ReceiverTasklet

use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.

the class ExecutionPlan method createIfAbsentReceiverTasklet.

private void createIfAbsentReceiverTasklet(EdgeDef edge, int[][] ptionsPerProcessor, int totalPtionCount) {
    final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.get(edge.edgeId());
    receiverMap.computeIfAbsent(edge.destVertex().vertexId(), x -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), x -> {
        Map<Address, ReceiverTasklet> addrToTasklet = new HashMap<>();
        // create a receiver per address
        int offset = 0;
        for (Address addr : ptionArrgmt.remotePartitionAssignment.get().keySet()) {
            final OutboundCollector[] collectors = new OutboundCollector[ptionsPerProcessor.length];
            // assign the queues starting from end
            final int queueOffset = --offset;
            Arrays.setAll(collectors, n -> new ConveyorCollector(localConveyors[n], localConveyors[n].queueCount() + queueOffset, ptionsPerProcessor[n]));
            final OutboundCollector collector = compositeCollector(collectors, edge, totalPtionCount);
            ReceiverTasklet receiverTasklet = new ReceiverTasklet(collector, edge.getConfig().getReceiveWindowMultiplier(), getConfig().getInstanceConfig().getFlowControlPeriodMs());
            addrToTasklet.put(addr, receiverTasklet);
        }
        return addrToTasklet;
    });
}
Also used : Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.nio.Address) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) Util.writeList(com.hazelcast.jet.impl.util.Util.writeList) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Util.readList(com.hazelcast.jet.impl.util.Util.readList) List(java.util.List) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) IntStream(java.util.stream.IntStream) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) IPartitionService(com.hazelcast.spi.partition.IPartitionService) IOException(java.io.IOException) NodeEngine(com.hazelcast.spi.NodeEngine) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetService(com.hazelcast.jet.impl.JetService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Address(com.hazelcast.nio.Address) HashMap(java.util.HashMap) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector)

Example 2 with ReceiverTasklet

use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.

the class ExecutionPlan method createOutboundCollectors.

private OutboundCollector[] createOutboundCollectors(EdgeDef edge, int processorIndex, Map<Address, ConcurrentConveyor<Object>> senderConveyorMap) {
    final int upstreamParallelism = edge.sourceVertex().localParallelism();
    final int downstreamParallelism = edge.destVertex().localParallelism();
    final int numRemoteMembers = ptionArrgmt.remotePartitionAssignment.get().size();
    final int queueSize = edge.getConfig().getQueueSize();
    final int[][] ptionsPerProcessor = ptionArrgmt.assignPartitionsToProcessors(downstreamParallelism, edge.isDistributed());
    if (edge.routingPolicy() == RoutingPolicy.ISOLATED) {
        if (downstreamParallelism < upstreamParallelism) {
            throw new IllegalArgumentException(String.format("The edge %s specifies the %s routing policy, but the downstream vertex" + " parallelism (%d) is less than the upstream vertex parallelism (%d)", edge, RoutingPolicy.ISOLATED.name(), downstreamParallelism, upstreamParallelism));
        }
        if (edge.isDistributed()) {
            throw new IllegalArgumentException("Isolated edges must be local: " + edge);
        }
        // there is only one producer per consumer for a one to many edge, so queueCount is always 1
        ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> createConveyorArray(downstreamParallelism, 1, queueSize));
        return IntStream.range(0, downstreamParallelism).filter(i -> i % upstreamParallelism == processorIndex).mapToObj(i -> new ConveyorCollector(localConveyors[i], 0, ptionsPerProcessor[i])).toArray(OutboundCollector[]::new);
    }
    /*
         * Each edge is represented by an array of conveyors between the producers and consumers
         * There are as many conveyors as there are consumers.
         * Each conveyor has one queue per producer.
         *
         * For a distributed edge, there is one additional producer per member represented
         * by the ReceiverTasklet.
         */
    final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> {
        int queueCount = upstreamParallelism + (edge.isDistributed() ? numRemoteMembers : 0);
        return createConveyorArray(downstreamParallelism, queueCount, queueSize);
    });
    final OutboundCollector[] localCollectors = new OutboundCollector[downstreamParallelism];
    Arrays.setAll(localCollectors, n -> new ConveyorCollector(localConveyors[n], processorIndex, ptionsPerProcessor[n]));
    // in a local edge, we only have the local collectors.
    if (!edge.isDistributed()) {
        return localCollectors;
    }
    // in a distributed edge, allCollectors[0] is the composite of local collectors, and
    // allCollectors[n] where n > 0 is a collector pointing to a remote member _n_.
    final int totalPtionCount = nodeEngine.getPartitionService().getPartitionCount();
    final OutboundCollector[] allCollectors;
    createIfAbsentReceiverTasklet(edge, ptionsPerProcessor, totalPtionCount);
    // assign remote partitions to outbound data collectors
    final Map<Address, int[]> memberToPartitions = ptionArrgmt.remotePartitionAssignment.get();
    allCollectors = new OutboundCollector[memberToPartitions.size() + 1];
    allCollectors[0] = compositeCollector(localCollectors, edge, totalPtionCount);
    int index = 1;
    for (Map.Entry<Address, int[]> entry : memberToPartitions.entrySet()) {
        allCollectors[index++] = new ConveyorCollectorWithPartition(senderConveyorMap.get(entry.getKey()), processorIndex, entry.getValue());
    }
    return allCollectors;
}
Also used : Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.nio.Address) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) Util.writeList(com.hazelcast.jet.impl.util.Util.writeList) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Util.readList(com.hazelcast.jet.impl.util.Util.readList) List(java.util.List) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) IntStream(java.util.stream.IntStream) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) IPartitionService(com.hazelcast.spi.partition.IPartitionService) IOException(java.io.IOException) NodeEngine(com.hazelcast.spi.NodeEngine) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetService(com.hazelcast.jet.impl.JetService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Address(com.hazelcast.nio.Address) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) HashMap(java.util.HashMap)

Example 3 with ReceiverTasklet

use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast by hazelcast.

the class Networking method createFlowControlPacket.

private Map<Address, byte[]> createFlowControlPacket() throws IOException {
    class MemberData {

        final BufferObjectDataOutput output = createObjectDataOutput(nodeEngine, lastFlowPacketSize);

        final Connection memberConnection;

        Long startedExecutionId;

        MemberData(Address address) {
            memberConnection = getMemberConnection(nodeEngine, address);
        }
    }
    Map<Address, MemberData> res = new HashMap<>();
    for (ExecutionContext execCtx : jobExecutionService.getExecutionContexts()) {
        Map<SenderReceiverKey, ReceiverTasklet> receiverMap = execCtx.receiverMap();
        if (receiverMap == null) {
            continue;
        }
        for (Entry<SenderReceiverKey, ReceiverTasklet> en : receiverMap.entrySet()) {
            assert !en.getKey().address.equals(nodeEngine.getThisAddress());
            MemberData md = res.computeIfAbsent(en.getKey().address, address -> new MemberData(address));
            if (md.startedExecutionId == null) {
                md.startedExecutionId = execCtx.executionId();
                md.output.writeLong(md.startedExecutionId);
            }
            assert en.getKey().vertexId != TERMINAL_VERTEX_ID;
            md.output.writeInt(en.getKey().vertexId);
            md.output.writeInt(en.getKey().ordinal);
            md.output.writeInt(en.getValue().updateAndGetSendSeqLimitCompressed(md.memberConnection));
        }
        for (MemberData md : res.values()) {
            if (md.startedExecutionId != null) {
                // write a mark to terminate values for an execution
                md.output.writeInt(TERMINAL_VERTEX_ID);
                md.startedExecutionId = null;
            }
        }
    }
    for (MemberData md : res.values()) {
        assert md.output.position() > 0;
        // write a mark to terminate all executions
        // Execution IDs are generated using Flake ID generator and those are >0 normally, we
        // use MIN_VALUE as a terminator.
        md.output.writeLong(TERMINAL_EXECUTION_ID);
    }
    // finalize the packets
    int maxSize = 0;
    for (Entry<Address, MemberData> entry : res.entrySet()) {
        byte[] data = entry.getValue().output.toByteArray();
        // we break type safety to avoid creating a new map, we replace the values to a different type in place
        @SuppressWarnings({ "unchecked", "rawtypes" }) Entry<Address, byte[]> entry1 = (Entry) entry;
        entry1.setValue(data);
        if (data.length > maxSize) {
            maxSize = data.length;
        }
    }
    lastFlowPacketSize = maxSize;
    return (Map) res;
}
Also used : Address(com.hazelcast.cluster.Address) HashMap(java.util.HashMap) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Connection(com.hazelcast.internal.nio.Connection) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) BufferObjectDataOutput(com.hazelcast.internal.nio.BufferObjectDataOutput) Entry(java.util.Map.Entry) ExecutionContext(com.hazelcast.jet.impl.execution.ExecutionContext) SenderReceiverKey(com.hazelcast.jet.impl.execution.ExecutionContext.SenderReceiverKey) HashMap(java.util.HashMap) Map(java.util.Map)

Example 4 with ReceiverTasklet

use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast by hazelcast.

the class ExecutionPlan method initialize.

/**
 * A method called on the members as part of the InitExecutionOperation.
 * Creates tasklets, inboxes/outboxes and connects these to make them ready
 * for a later StartExecutionOperation.
 */
public void initialize(NodeEngineImpl nodeEngine, long jobId, long executionId, @Nonnull SnapshotContext snapshotContext, ConcurrentHashMap<String, File> tempDirectories, InternalSerializationService jobSerializationService) {
    this.nodeEngine = nodeEngine;
    this.jobClassLoaderService = ((JetServiceBackend) nodeEngine.getService(JetServiceBackend.SERVICE_NAME)).getJobClassLoaderService();
    this.executionId = executionId;
    initProcSuppliers(jobId, tempDirectories, jobSerializationService);
    initDag(jobSerializationService);
    this.ptionArrgmt = new PartitionArrangement(partitionAssignment, nodeEngine.getThisAddress());
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(vertices);
    for (Address destAddr : remoteMembers.get()) {
        Connection conn = getMemberConnection(nodeEngine, destAddr);
        if (conn == null) {
            throw new TopologyChangedException("no connection to job participant: " + destAddr);
        }
        memberConnections.put(destAddr, conn);
    }
    for (VertexDef vertex : vertices) {
        ClassLoader processorClassLoader = isLightJob ? null : jobClassLoaderService.getProcessorClassLoader(jobId, vertex.name());
        Collection<? extends Processor> processors = doWithClassLoader(processorClassLoader, () -> createProcessors(vertex, vertex.localParallelism()));
        String jobPrefix = prefix(jobConfig.getName(), jobId, vertex.name());
        // create StoreSnapshotTasklet and the queues to it
        ConcurrentConveyor<Object> ssConveyor = null;
        if (!isLightJob) {
            // Note that we create the snapshot queues for all non-light jobs, even if they don't have
            // processing guarantee enabled, because in EE one can request a snapshot also for
            // non-snapshotted jobs.
            @SuppressWarnings("unchecked") QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
            Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
            ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
            ILogger storeSnapshotLogger = prefixedLogger(nodeEngine.getLogger(StoreSnapshotTasklet.class), jobPrefix);
            StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, ConcurrentInboundEdgeStream.create(ssConveyor, 0, 0, true, jobPrefix + "/ssFrom", null), new AsyncSnapshotWriterImpl(nodeEngine, snapshotContext, vertex.name(), memberIndex, memberCount, jobSerializationService), storeSnapshotLogger, vertex.name(), higherPriorityVertices.contains(vertex.vertexId()));
            tasklets.add(ssTasklet);
        }
        int localProcessorIdx = 0;
        for (Processor processor : processors) {
            int globalProcessorIndex = memberIndex * vertex.localParallelism() + localProcessorIdx;
            String processorPrefix = prefix(jobConfig.getName(), jobId, vertex.name(), globalProcessorIndex);
            ILogger logger = prefixedLogger(nodeEngine.getLogger(processor.getClass()), processorPrefix);
            ProcCtx context = new ProcCtx(nodeEngine, jobId, executionId, getJobConfig(), logger, vertex.name(), localProcessorIdx, globalProcessorIndex, isLightJob, partitionAssignment, vertex.localParallelism(), memberIndex, memberCount, tempDirectories, jobSerializationService, subject, processorClassLoader);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, jobSerializationService);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, globalProcessorIndex);
            OutboundCollector snapshotCollector = ssConveyor == null ? null : new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            // vertices which are only used for snapshot restore will not be marked as "source=true" in metrics
            // also do not consider snapshot restore edges for determining source tag
            boolean isSource = vertex.inboundEdges().stream().allMatch(EdgeDef::isSnapshotRestoreEdge) && !vertex.isSnapshotVertex();
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, nodeEngine.getExecutionService().getExecutor(TASKLET_INIT_CLOSE_EXECUTOR_NAME), jobSerializationService, processor, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, isSource);
            tasklets.add(processorTasklet);
            this.processors.add(processor);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}
Also used : Address(com.hazelcast.cluster.Address) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) ObjectWithPartitionId(com.hazelcast.jet.impl.util.ObjectWithPartitionId) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) ImdgUtil(com.hazelcast.jet.impl.util.ImdgUtil) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) SerializationServiceAware(com.hazelcast.internal.serialization.SerializationServiceAware) DISTRIBUTE_TO_ALL(com.hazelcast.jet.core.Edge.DISTRIBUTE_TO_ALL) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) TASKLET_INIT_CLOSE_EXECUTOR_NAME(com.hazelcast.jet.impl.execution.TaskletExecutionService.TASKLET_INIT_CLOSE_EXECUTOR_NAME) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) PrefixedLogger.prefix(com.hazelcast.jet.impl.util.PrefixedLogger.prefix) Collection(java.util.Collection) IPartitionService(com.hazelcast.internal.partition.IPartitionService) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) IntStream(java.util.stream.IntStream) ComparatorEx(com.hazelcast.function.ComparatorEx) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) ImdgUtil.writeList(com.hazelcast.jet.impl.util.ImdgUtil.writeList) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) PrefixedLogger.prefixedLogger(com.hazelcast.jet.impl.util.PrefixedLogger.prefixedLogger) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) IntFunction(java.util.function.IntFunction) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Connection(com.hazelcast.internal.nio.Connection) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) AsyncSnapshotWriterImpl(com.hazelcast.jet.impl.util.AsyncSnapshotWriterImpl) IOException(java.io.IOException) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Subject(javax.security.auth.Subject) File(java.io.File) ImdgUtil.readList(com.hazelcast.jet.impl.util.ImdgUtil.readList) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.cluster.Address) AsyncSnapshotWriterImpl(com.hazelcast.jet.impl.util.AsyncSnapshotWriterImpl) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) ILogger(com.hazelcast.logging.ILogger) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Connection(com.hazelcast.internal.nio.Connection) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe)

Example 5 with ReceiverTasklet

use of com.hazelcast.jet.impl.execution.ReceiverTasklet in project hazelcast-jet by hazelcast.

the class ExecutionPlan method initialize.

public void initialize(NodeEngine nodeEngine, long jobId, long executionId, SnapshotContext snapshotContext) {
    this.nodeEngine = nodeEngine;
    this.executionId = executionId;
    initProcSuppliers();
    initDag();
    this.ptionArrgmt = new PartitionArrangement(partitionOwners, nodeEngine.getThisAddress());
    JetInstance instance = getJetInstance(nodeEngine);
    for (VertexDef vertex : vertices) {
        Collection<? extends Processor> processors = createProcessors(vertex, vertex.localParallelism());
        // create StoreSnapshotTasklet and the queues to it
        QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
        Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
        ConcurrentConveyor<Object> ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
        StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, jobId, new ConcurrentInboundEdgeStream(ssConveyor, 0, 0, lastSnapshotId, true, -1, "ssFrom:" + vertex.name()), nodeEngine, vertex.name(), vertex.isHigherPriorityUpstream());
        tasklets.add(ssTasklet);
        int localProcessorIdx = 0;
        for (Processor p : processors) {
            int globalProcessorIndex = vertex.getProcIdxOffset() + localProcessorIdx;
            String loggerName = createLoggerName(p.getClass().getName(), vertex.name(), globalProcessorIndex);
            ProcCtx context = new ProcCtx(instance, nodeEngine.getSerializationService(), nodeEngine.getLogger(loggerName), vertex.name(), globalProcessorIndex, jobConfig.getProcessingGuarantee(), vertex.localParallelism(), vertex.totalParallelism());
            String probePrefix = String.format("jet.job.%s.%s#%d", idToString(executionId), vertex.name(), localProcessorIdx);
            ((NodeEngineImpl) nodeEngine).getMetricsRegistry().scanAndRegister(p, probePrefix);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx);
            OutboundCollector snapshotCollector = new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, p, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, jobConfig.getMaxWatermarkRetainMillis());
            tasklets.add(processorTasklet);
            this.processors.add(p);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}
Also used : Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.nio.Address) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) Util.writeList(com.hazelcast.jet.impl.util.Util.writeList) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) Collection(java.util.Collection) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Util.readList(com.hazelcast.jet.impl.util.Util.readList) List(java.util.List) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) IntStream(java.util.stream.IntStream) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) IPartitionService(com.hazelcast.spi.partition.IPartitionService) IOException(java.io.IOException) NodeEngine(com.hazelcast.spi.NodeEngine) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetService(com.hazelcast.jet.impl.JetService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Processor(com.hazelcast.jet.core.Processor) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) Util.idToString(com.hazelcast.jet.impl.util.Util.idToString) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) Util.getJetInstance(com.hazelcast.jet.impl.util.Util.getJetInstance) JetInstance(com.hazelcast.jet.JetInstance) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe)

Aggregations

ReceiverTasklet (com.hazelcast.jet.impl.execution.ReceiverTasklet)6 HashMap (java.util.HashMap)6 Map (java.util.Map)6 ConcurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor)5 ConcurrentConveyor.concurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor)5 OneToOneConcurrentArrayQueue (com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue)5 QueuedPipe (com.hazelcast.internal.util.concurrent.QueuedPipe)5 JetException (com.hazelcast.jet.JetException)5 DEFAULT_QUEUE_SIZE (com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE)5 JetConfig (com.hazelcast.jet.config.JetConfig)5 JobConfig (com.hazelcast.jet.config.JobConfig)5 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)5 RoutingPolicy (com.hazelcast.jet.core.Edge.RoutingPolicy)5 Processor (com.hazelcast.jet.core.Processor)5 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)5 ConcurrentInboundEdgeStream (com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream)5 ConveyorCollector (com.hazelcast.jet.impl.execution.ConveyorCollector)5 ConveyorCollectorWithPartition (com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition)5 InboundEdgeStream (com.hazelcast.jet.impl.execution.InboundEdgeStream)5 OutboundCollector (com.hazelcast.jet.impl.execution.OutboundCollector)5