Examples with PartitionDescriptor org.apache.flink.runtime.shuffle.PartitionDescriptor used on opensource projects

Example 1 with PartitionDescriptor

use of org.apache.flink.runtime.shuffle.PartitionDescriptor in project flink by apache.

the class TaskExecutorSubmissionTest method createSender.

private TaskDeploymentDescriptor createSender(NettyShuffleDescriptor shuffleDescriptor, Class<? extends AbstractInvokable> abstractInvokable) throws IOException {
    PartitionDescriptor partitionDescriptor = PartitionDescriptorBuilder.newBuilder().setPartitionId(shuffleDescriptor.getResultPartitionID().getPartitionId()).build();
    ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor = new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1, true);
    return createTestTaskDeploymentDescriptor("Sender", shuffleDescriptor.getResultPartitionID().getProducerId(), abstractInvokable, 1, Collections.singletonList(resultPartitionDeploymentDescriptor), Collections.emptyList());
}

Example 2 with PartitionDescriptor

use of org.apache.flink.runtime.shuffle.PartitionDescriptor in project flink by apache.

the class TaskTest method testExecutionFailsInNetworkRegistrationForPartitions.

@Test
public void testExecutionFailsInNetworkRegistrationForPartitions() throws Exception {
    final PartitionDescriptor partitionDescriptor = PartitionDescriptorBuilder.newBuilder().build();
    final ShuffleDescriptor shuffleDescriptor = NettyShuffleDescriptorBuilder.newBuilder().buildLocal();
    final ResultPartitionDeploymentDescriptor dummyPartition = new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1, false);
    testExecutionFailsInNetworkRegistration(Collections.singletonList(dummyPartition), Collections.emptyList());
}

Example 3 with PartitionDescriptor

use of org.apache.flink.runtime.shuffle.PartitionDescriptor in project flink by apache.

the class ShuffleDescriptorTest method createResultPartitionDeploymentDescriptor.

private static ResultPartitionDeploymentDescriptor createResultPartitionDeploymentDescriptor(JobID jobID, ResultPartitionID id, ResourceID location) throws ExecutionException, InterruptedException {
    ProducerDescriptor producerDescriptor = new ProducerDescriptor(location, id.getProducerId(), STUB_CONNECTION_ID.getAddress().getAddress(), STUB_CONNECTION_ID.getAddress().getPort());
    PartitionDescriptor partitionDescriptor = PartitionDescriptorBuilder.newBuilder().setPartitionId(id.getPartitionId()).build();
    ShuffleDescriptor shuffleDescriptor = ShuffleTestUtils.DEFAULT_SHUFFLE_MASTER.registerPartitionWithProducer(jobID, partitionDescriptor, producerDescriptor).get();
    return new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1, true);
}

Example 4 with PartitionDescriptor

use of org.apache.flink.runtime.shuffle.PartitionDescriptor in project flink by apache.

the class Execution method registerProducedPartitions.

@VisibleForTesting
static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions(ExecutionVertex vertex, TaskManagerLocation location, ExecutionAttemptID attemptId, boolean notifyPartitionDataAvailable) {
    ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId);
    Collection<IntermediateResultPartition> partitions = vertex.getProducedPartitions().values();
    Collection<CompletableFuture<ResultPartitionDeploymentDescriptor>> partitionRegistrations = new ArrayList<>(partitions.size());
    for (IntermediateResultPartition partition : partitions) {
        PartitionDescriptor partitionDescriptor = PartitionDescriptor.from(partition);
        int maxParallelism = getPartitionMaxParallelism(partition);
        CompletableFuture<? extends ShuffleDescriptor> shuffleDescriptorFuture = vertex.getExecutionGraphAccessor().getShuffleMaster().registerPartitionWithProducer(vertex.getJobId(), partitionDescriptor, producerDescriptor);
        CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture.thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, maxParallelism, notifyPartitionDataAvailable));
        partitionRegistrations.add(partitionRegistration);
    }
    return FutureUtils.combineAll(partitionRegistrations).thenApply(rpdds -> {
        Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor> producedPartitions = new LinkedHashMap<>(partitions.size());
        rpdds.forEach(rpdd -> producedPartitions.put(rpdd.getPartitionId(), rpdd));
        return producedPartitions;
    });
}