Examples with ConveyorCollectorWithPartition com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition used on opensource projects

Example 1 with ConveyorCollectorWithPartition

use of com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition in project hazelcast-jet by hazelcast.

the class ExecutionPlan method createOutboundCollectors.

private OutboundCollector[] createOutboundCollectors(EdgeDef edge, int processorIndex, Map<Address, ConcurrentConveyor<Object>> senderConveyorMap) {
    final int upstreamParallelism = edge.sourceVertex().localParallelism();
    final int downstreamParallelism = edge.destVertex().localParallelism();
    final int numRemoteMembers = ptionArrgmt.remotePartitionAssignment.get().size();
    final int queueSize = edge.getConfig().getQueueSize();
    final int[][] ptionsPerProcessor = ptionArrgmt.assignPartitionsToProcessors(downstreamParallelism, edge.isDistributed());
    if (edge.routingPolicy() == RoutingPolicy.ISOLATED) {
        if (downstreamParallelism < upstreamParallelism) {
            throw new IllegalArgumentException(String.format("The edge %s specifies the %s routing policy, but the downstream vertex" + " parallelism (%d) is less than the upstream vertex parallelism (%d)", edge, RoutingPolicy.ISOLATED.name(), downstreamParallelism, upstreamParallelism));
        }
        if (edge.isDistributed()) {
            throw new IllegalArgumentException("Isolated edges must be local: " + edge);
        }
        // there is only one producer per consumer for a one to many edge, so queueCount is always 1
        ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> createConveyorArray(downstreamParallelism, 1, queueSize));
        return IntStream.range(0, downstreamParallelism).filter(i -> i % upstreamParallelism == processorIndex).mapToObj(i -> new ConveyorCollector(localConveyors[i], 0, ptionsPerProcessor[i])).toArray(OutboundCollector[]::new);
    }
    /*
         * Each edge is represented by an array of conveyors between the producers and consumers
         * There are as many conveyors as there are consumers.
         * Each conveyor has one queue per producer.
         *
         * For a distributed edge, there is one additional producer per member represented
         * by the ReceiverTasklet.
         */
    final ConcurrentConveyor<Object>[] localConveyors = localConveyorMap.computeIfAbsent(edge.edgeId(), e -> {
        int queueCount = upstreamParallelism + (edge.isDistributed() ? numRemoteMembers : 0);
        return createConveyorArray(downstreamParallelism, queueCount, queueSize);
    });
    final OutboundCollector[] localCollectors = new OutboundCollector[downstreamParallelism];
    Arrays.setAll(localCollectors, n -> new ConveyorCollector(localConveyors[n], processorIndex, ptionsPerProcessor[n]));
    // in a local edge, we only have the local collectors.
    if (!edge.isDistributed()) {
        return localCollectors;
    }
    // in a distributed edge, allCollectors[0] is the composite of local collectors, and
    // allCollectors[n] where n > 0 is a collector pointing to a remote member _n_.
    final int totalPtionCount = nodeEngine.getPartitionService().getPartitionCount();
    final OutboundCollector[] allCollectors;
    createIfAbsentReceiverTasklet(edge, ptionsPerProcessor, totalPtionCount);
    // assign remote partitions to outbound data collectors
    final Map<Address, int[]> memberToPartitions = ptionArrgmt.remotePartitionAssignment.get();
    allCollectors = new OutboundCollector[memberToPartitions.size() + 1];
    allCollectors[0] = compositeCollector(localCollectors, edge, totalPtionCount);
    int index = 1;
    for (Map.Entry<Address, int[]> entry : memberToPartitions.entrySet()) {
        allCollectors[index++] = new ConveyorCollectorWithPartition(senderConveyorMap.get(entry.getKey()), processorIndex, entry.getValue());
    }
    return allCollectors;
}

Example 2 with ConveyorCollectorWithPartition

use of com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition in project hazelcast by hazelcast.

the class ExecutionPlan method createRemoteOutboundCollectors.

private OutboundCollector[] createRemoteOutboundCollectors(EdgeDef edge, String jobPrefix, int processorIndex, int totalPartitionCount, int[][] partitionsPerProcessor, InternalSerializationService jobSerializationService) {
    // the distributed-to-one edge must be partitioned and the target member must be present
    if (!edge.getDistributedTo().equals(DISTRIBUTE_TO_ALL)) {
        if (edge.routingPolicy() != RoutingPolicy.PARTITIONED) {
            throw new JetException("An edge distributing to a specific member must be partitioned: " + edge);
        }
        if (!ptionArrgmt.getRemotePartitionAssignment().containsKey(edge.getDistributedTo()) && !edge.getDistributedTo().equals(nodeEngine.getThisAddress())) {
            throw new JetException("The target member of an edge is not present in the cluster or is a lite member: " + edge);
        }
    }
    Map<Address, ConcurrentConveyor<Object>> senderConveyorMap = memberToSenderConveyorMap(edgeSenderConveyorMap, edge, jobPrefix, jobSerializationService);
    createIfAbsentReceiverTasklet(edge, jobPrefix, partitionsPerProcessor, totalPartitionCount, jobSerializationService);
    // assign remote partitions to outbound data collectors
    Address distributeTo = edge.getDistributedTo();
    Map<Address, int[]> memberToPartitions = distributeTo.equals(DISTRIBUTE_TO_ALL) ? ptionArrgmt.getRemotePartitionAssignment() : ptionArrgmt.remotePartitionAssignmentToOne(distributeTo);
    OutboundCollector[] remoteCollectors = new OutboundCollector[memberToPartitions.size()];
    int index = 0;
    for (Map.Entry<Address, int[]> entry : memberToPartitions.entrySet()) {
        Address memberAddress = entry.getKey();
        int[] memberPartitions = entry.getValue();
        ConcurrentConveyor<Object> conveyor = senderConveyorMap.get(memberAddress);
        remoteCollectors[index++] = new ConveyorCollectorWithPartition(conveyor, processorIndex, memberPartitions);
    }
    return remoteCollectors;
}