Examples with ShuffleDescriptor org.apache.flink.runtime.shuffle.ShuffleDescriptor used on opensource projects

Example 16 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ResultPartitionDeploymentDescriptorTest method testSerializationWithNettyShuffleDescriptor.

/**
 * Tests simple de/serialization with {@link NettyShuffleDescriptor}.
 */
@Test
public void testSerializationWithNettyShuffleDescriptor() throws IOException {
    ShuffleDescriptor shuffleDescriptor = new NettyShuffleDescriptor(producerLocation, new NetworkPartitionConnectionInfo(connectionID), resultPartitionID);
    ResultPartitionDeploymentDescriptor copy = createCopyAndVerifyResultPartitionDeploymentDescriptor(shuffleDescriptor);
    assertThat(copy.getShuffleDescriptor(), instanceOf(NettyShuffleDescriptor.class));
    NettyShuffleDescriptor shuffleDescriptorCopy = (NettyShuffleDescriptor) copy.getShuffleDescriptor();
    assertThat(shuffleDescriptorCopy.getResultPartitionID(), is(resultPartitionID));
    assertThat(shuffleDescriptorCopy.isUnknown(), is(false));
    assertThat(shuffleDescriptorCopy.isLocalTo(producerLocation), is(true));
    assertThat(shuffleDescriptorCopy.getConnectionId(), is(connectionID));
}

Example 17 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method getConsumedPartitionShuffleDescriptor.

private static ShuffleDescriptor getConsumedPartitionShuffleDescriptor(ResultPartitionID id, ExecutionState state, @Nullable ResultPartitionDeploymentDescriptor producedPartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint partitionLocationConstraint) {
    ShuffleDescriptor shuffleDescriptor = TaskDeploymentDescriptorFactory.getConsumedPartitionShuffleDescriptor(id, ResultPartitionType.PIPELINED, true, state, partitionLocationConstraint, producedPartition);
    assertThat(shuffleDescriptor, is(notNullValue()));
    assertThat(shuffleDescriptor.getResultPartitionID(), is(id));
    return shuffleDescriptor;
}

Example 18 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method createResultPartitionDeploymentDescriptor.

private static ResultPartitionDeploymentDescriptor createResultPartitionDeploymentDescriptor(JobID jobID, ResultPartitionID id, ResourceID location) throws ExecutionException, InterruptedException {
    ProducerDescriptor producerDescriptor = new ProducerDescriptor(location, id.getProducerId(), STUB_CONNECTION_ID.getAddress().getAddress(), STUB_CONNECTION_ID.getAddress().getPort());
    PartitionDescriptor partitionDescriptor = PartitionDescriptorBuilder.newBuilder().setPartitionId(id.getPartitionId()).build();
    ShuffleDescriptor shuffleDescriptor = ShuffleTestUtils.DEFAULT_SHUFFLE_MASTER.registerPartitionWithProducer(jobID, partitionDescriptor, producerDescriptor).get();
    return new ResultPartitionDeploymentDescriptor(partitionDescriptor, shuffleDescriptor, 1, true);
}

Example 19 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class TaskDeploymentDescriptorFactoryTest method testCacheShuffleDescriptor.

private void testCacheShuffleDescriptor(TestingBlobWriter blobWriter) throws Exception {
    final JobID jobId = new JobID();
    final ExecutionJobVertex ejv = setupExecutionGraphAndGetVertex(jobId, blobWriter);
    final ExecutionVertex ev21 = ejv.getTaskVertices()[0];
    createTaskDeploymentDescriptor(ev21);
    // The ShuffleDescriptors should be cached
    final IntermediateResult consumedResult = ejv.getInputs().get(0);
    final MaybeOffloaded<ShuffleDescriptor[]> maybeOffloaded = consumedResult.getCachedShuffleDescriptors(ev21.getConsumedPartitionGroup(0));
    final ShuffleDescriptor[] cachedShuffleDescriptors = deserializeShuffleDescriptors(maybeOffloaded, jobId, blobWriter);
    // Check if the ShuffleDescriptors are cached correctly
    assertEquals(ev21.getConsumedPartitionGroup(0).size(), cachedShuffleDescriptors.length);
    int idx = 0;
    for (IntermediateResultPartitionID consumedPartitionId : ev21.getConsumedPartitionGroup(0)) {
        assertEquals(consumedPartitionId, cachedShuffleDescriptors[idx++].getResultPartitionID().getPartitionId());
    }
}

Example 20 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class SingleInputGateTest method createInputGateWithLocalChannels.

private static Map<InputGateID, SingleInputGate> createInputGateWithLocalChannels(NettyShuffleEnvironment network, int numberOfGates, @SuppressWarnings("SameParameterValue") int numberOfLocalChannels) throws IOException {
    ShuffleDescriptor[] channelDescs = new NettyShuffleDescriptor[numberOfLocalChannels];
    for (int i = 0; i < numberOfLocalChannels; i++) {
        channelDescs[i] = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
    }
    InputGateDeploymentDescriptor[] gateDescs = new InputGateDeploymentDescriptor[numberOfGates];
    IntermediateDataSetID[] ids = new IntermediateDataSetID[numberOfGates];
    for (int i = 0; i < numberOfGates; i++) {
        ids[i] = new IntermediateDataSetID();
        gateDescs[i] = new InputGateDeploymentDescriptor(ids[i], ResultPartitionType.PIPELINED, 0, channelDescs);
    }
    ExecutionAttemptID consumerID = new ExecutionAttemptID();
    SingleInputGate[] gates = network.createInputGates(network.createShuffleIOOwnerContext("", consumerID, new UnregisteredMetricsGroup()), SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER, asList(gateDescs)).toArray(new SingleInputGate[] {});
    Map<InputGateID, SingleInputGate> inputGatesById = new HashMap<>();
    for (int i = 0; i < numberOfGates; i++) {
        inputGatesById.put(new InputGateID(ids[i], consumerID), gates[i]);
    }
    return inputGatesById;
}