Search in sources :

Example 1 with ComparatorEx

use of com.hazelcast.function.ComparatorEx in project hazelcast by hazelcast.

the class ExecutionPlan method memberToSenderConveyorMap.

/**
 * Creates (if absent) for the given edge one sender tasklet per remote member,
 * each with a single conveyor with a number of producer queues feeding it.
 * Populates the {@link #senderMap} and {@link #tasklets} fields.
 */
private Map<Address, ConcurrentConveyor<Object>> memberToSenderConveyorMap(Map<String, Map<Address, ConcurrentConveyor<Object>>> edgeSenderConveyorMap, EdgeDef edge, String jobPrefix, InternalSerializationService jobSerializationService) {
    assert !edge.isLocal() : "Edge is not distributed";
    return edgeSenderConveyorMap.computeIfAbsent(edge.edgeId(), x -> {
        final Map<Address, ConcurrentConveyor<Object>> addrToConveyor = new HashMap<>();
        for (Address destAddr : remoteMembers.get()) {
            final ConcurrentConveyor<Object> conveyor = createConveyorArray(1, edge.sourceVertex().localParallelism(), edge.getConfig().getQueueSize())[0];
            @SuppressWarnings("unchecked") ComparatorEx<Object> origComparator = (ComparatorEx<Object>) edge.getOrderComparator();
            ComparatorEx<ObjectWithPartitionId> adaptedComparator = origComparator == null ? null : (l, r) -> origComparator.compare(l.getItem(), r.getItem());
            final InboundEdgeStream inboundEdgeStream = newEdgeStream(edge, conveyor, jobPrefix + "/toVertex:" + edge.destVertex().name() + "-toMember:" + destAddr, adaptedComparator);
            final int destVertexId = edge.destVertex().vertexId();
            final SenderTasklet t = new SenderTasklet(inboundEdgeStream, nodeEngine, destAddr, memberConnections.get(destAddr), destVertexId, edge.getConfig().getPacketSizeLimit(), executionId, edge.sourceVertex().name(), edge.sourceOrdinal(), jobSerializationService);
            senderMap.computeIfAbsent(destVertexId, xx -> new HashMap<>()).computeIfAbsent(edge.destOrdinal(), xx -> new HashMap<>()).put(destAddr, t);
            tasklets.add(t);
            addrToConveyor.put(destAddr, conveyor);
        }
        return addrToConveyor;
    });
}
Also used : Address(com.hazelcast.cluster.Address) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) ObjectWithPartitionId(com.hazelcast.jet.impl.util.ObjectWithPartitionId) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) ImdgUtil(com.hazelcast.jet.impl.util.ImdgUtil) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) SerializationServiceAware(com.hazelcast.internal.serialization.SerializationServiceAware) DISTRIBUTE_TO_ALL(com.hazelcast.jet.core.Edge.DISTRIBUTE_TO_ALL) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) TASKLET_INIT_CLOSE_EXECUTOR_NAME(com.hazelcast.jet.impl.execution.TaskletExecutionService.TASKLET_INIT_CLOSE_EXECUTOR_NAME) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) PrefixedLogger.prefix(com.hazelcast.jet.impl.util.PrefixedLogger.prefix) Collection(java.util.Collection) IPartitionService(com.hazelcast.internal.partition.IPartitionService) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) IntStream(java.util.stream.IntStream) ComparatorEx(com.hazelcast.function.ComparatorEx) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) ImdgUtil.writeList(com.hazelcast.jet.impl.util.ImdgUtil.writeList) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) PrefixedLogger.prefixedLogger(com.hazelcast.jet.impl.util.PrefixedLogger.prefixedLogger) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) IntFunction(java.util.function.IntFunction) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Connection(com.hazelcast.internal.nio.Connection) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) AsyncSnapshotWriterImpl(com.hazelcast.jet.impl.util.AsyncSnapshotWriterImpl) IOException(java.io.IOException) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Subject(javax.security.auth.Subject) File(java.io.File) ImdgUtil.readList(com.hazelcast.jet.impl.util.ImdgUtil.readList) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Address(com.hazelcast.cluster.Address) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) ObjectWithPartitionId(com.hazelcast.jet.impl.util.ObjectWithPartitionId) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) ComparatorEx(com.hazelcast.function.ComparatorEx)

Aggregations

Address (com.hazelcast.cluster.Address)1 ComparatorEx (com.hazelcast.function.ComparatorEx)1 Connection (com.hazelcast.internal.nio.Connection)1 IPartitionService (com.hazelcast.internal.partition.IPartitionService)1 InternalSerializationService (com.hazelcast.internal.serialization.InternalSerializationService)1 SerializationServiceAware (com.hazelcast.internal.serialization.SerializationServiceAware)1 ConcurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor)1 ConcurrentConveyor.concurrentConveyor (com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor)1 OneToOneConcurrentArrayQueue (com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue)1 QueuedPipe (com.hazelcast.internal.util.concurrent.QueuedPipe)1 JetException (com.hazelcast.jet.JetException)1 DEFAULT_QUEUE_SIZE (com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE)1 JetConfig (com.hazelcast.jet.config.JetConfig)1 JobConfig (com.hazelcast.jet.config.JobConfig)1 ProcessingGuarantee (com.hazelcast.jet.config.ProcessingGuarantee)1 DISTRIBUTE_TO_ALL (com.hazelcast.jet.core.Edge.DISTRIBUTE_TO_ALL)1 RoutingPolicy (com.hazelcast.jet.core.Edge.RoutingPolicy)1 Processor (com.hazelcast.jet.core.Processor)1 ProcessorSupplier (com.hazelcast.jet.core.ProcessorSupplier)1 TopologyChangedException (com.hazelcast.jet.core.TopologyChangedException)1