Examples with ProducerDescriptor org.apache.flink.runtime.shuffle.ProducerDescriptor used on opensource projects

Example 1 with ProducerDescriptor

use of org.apache.flink.runtime.shuffle.ProducerDescriptor in project flink by apache.

the class ShuffleDescriptorTest method createResultPartitionDeploymentDescriptor.

private static ResultPartitionDeploymentDescriptor createResultPartitionDeploymentDescriptor(JobID jobID, ResultPartitionID id, ResourceID location) throws ExecutionException, InterruptedException {
    ProducerDescriptor producerDescriptor = new ProducerDescriptor(location, id.getProducerId(), STUB_CONNECTION_ID.getAddress().getAddress(), STUB_CONNECTION_ID.getAddress().getPort());
    PartitionDescriptor partitionDescriptor = PartitionDescriptorBuilder.newBuilder().setPartitionId(id.getPartitionId()).build();
    ShuffleDescriptor shuffleDescriptor = ShuffleTestUtils.DEFAULT_SHUFFLE_MASTER.registerPartitionWithProducer(jobID, partitionDescriptor, producerDescriptor).get();
    return new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1, true);
}

Example 2 with ProducerDescriptor

use of org.apache.flink.runtime.shuffle.ProducerDescriptor in project flink by apache.

the class Execution method registerProducedPartitions.

@VisibleForTesting
static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions(ExecutionVertex vertex, TaskManagerLocation location, ExecutionAttemptID attemptId, boolean notifyPartitionDataAvailable) {
    ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId);
    Collection<IntermediateResultPartition> partitions = vertex.getProducedPartitions().values();
    Collection<CompletableFuture<ResultPartitionDeploymentDescriptor>> partitionRegistrations = new ArrayList<>(partitions.size());
    for (IntermediateResultPartition partition : partitions) {
        PartitionDescriptor partitionDescriptor = PartitionDescriptor.from(partition);
        int maxParallelism = getPartitionMaxParallelism(partition);
        CompletableFuture<? extends ShuffleDescriptor> shuffleDescriptorFuture = vertex.getExecutionGraphAccessor().getShuffleMaster().registerPartitionWithProducer(vertex.getJobId(), partitionDescriptor, producerDescriptor);
        CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture.thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, maxParallelism, notifyPartitionDataAvailable));
        partitionRegistrations.add(partitionRegistration);
    }
    return FutureUtils.combineAll(partitionRegistrations).thenApply(rpdds -> {
        Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor> producedPartitions = new LinkedHashMap<>(partitions.size());
        rpdds.forEach(rpdd -> producedPartitions.put(rpdd.getPartitionId(), rpdd));
        return producedPartitions;
    });
}