Examples with UnknownShuffleDescriptor org.apache.flink.runtime.shuffle.UnknownShuffleDescriptor used on opensource projects

Example 1 with UnknownShuffleDescriptor

use of org.apache.flink.runtime.shuffle.UnknownShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method testMixedLocalRemoteUnknownDeployment.

/**
 * Tests the deployment descriptors for local, remote, and unknown partition locations (with
 * lazy deployment allowed and all execution states for the producers).
 */
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
    ResourceID consumerResourceID = ResourceID.generate();
    JobID jobID = new JobID();
    // states.
    for (ExecutionState state : ExecutionState.values()) {
        ResultPartitionID localPartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor localPartition = createResultPartitionDeploymentDescriptor(jobID, localPartitionId, consumerResourceID);
        ResultPartitionID remotePartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor remotePartition = createResultPartitionDeploymentDescriptor(jobID, remotePartitionId, ResourceID.generate());
        ResultPartitionID unknownPartitionId = new ResultPartitionID();
        ShuffleDescriptor localShuffleDescriptor = getConsumedPartitionShuffleDescriptor(localPartitionId, state, localPartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor remoteShuffleDescriptor = getConsumedPartitionShuffleDescriptor(remotePartitionId, state, remotePartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor unknownShuffleDescriptor = getConsumedPartitionShuffleDescriptor(unknownPartitionId, state, null, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        // These states are allowed
        if (state == ExecutionState.RUNNING || state == ExecutionState.INITIALIZING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
            NettyShuffleDescriptor nettyShuffleDescriptor;
            // Create local or remote channels
            verifyShuffleDescriptor(localShuffleDescriptor, NettyShuffleDescriptor.class, false, localPartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) localShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(true));
            verifyShuffleDescriptor(remoteShuffleDescriptor, NettyShuffleDescriptor.class, false, remotePartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) remoteShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(false));
            assertThat(nettyShuffleDescriptor.getConnectionId(), is(STUB_CONNECTION_ID));
        } else {
            // Unknown (lazy deployment allowed)
            verifyShuffleDescriptor(localShuffleDescriptor, UnknownShuffleDescriptor.class, true, localPartitionId);
            verifyShuffleDescriptor(remoteShuffleDescriptor, UnknownShuffleDescriptor.class, true, remotePartitionId);
        }
        verifyShuffleDescriptor(unknownShuffleDescriptor, UnknownShuffleDescriptor.class, true, unknownPartitionId);
    }
}

Example 2 with UnknownShuffleDescriptor

use of org.apache.flink.runtime.shuffle.UnknownShuffleDescriptor in project flink by apache.

the class ResultPartitionDeploymentDescriptorTest method testSerializationOfUnknownShuffleDescriptor.

/**
 * Tests simple de/serialization with {@link UnknownShuffleDescriptor}.
 */
@Test
public void testSerializationOfUnknownShuffleDescriptor() throws IOException {
    ShuffleDescriptor shuffleDescriptor = new UnknownShuffleDescriptor(resultPartitionID);
    ShuffleDescriptor shuffleDescriptorCopy = CommonTestUtils.createCopySerializable(shuffleDescriptor);
    assertThat(shuffleDescriptorCopy, instanceOf(UnknownShuffleDescriptor.class));
    assertThat(shuffleDescriptorCopy.getResultPartitionID(), is(resultPartitionID));
    assertThat(shuffleDescriptorCopy.isUnknown(), is(true));
}

Example 3 with UnknownShuffleDescriptor

use of org.apache.flink.runtime.shuffle.UnknownShuffleDescriptor in project flink by apache.

the class SingleInputGateTest method createSingleInputGate.

static SingleInputGate createSingleInputGate(IntermediateResultPartitionID[] partitionIds, ResultPartitionType resultPartitionType, SubpartitionIndexRange subpartitionIndexRange, NettyShuffleEnvironment netEnv, ResourceID localLocation, ConnectionManager connectionManager, ResultPartitionManager resultPartitionManager) throws IOException {
    ShuffleDescriptor[] channelDescs = new ShuffleDescriptor[] { // Local
    createRemoteWithIdAndLocation(partitionIds[0], localLocation), // Remote
    createRemoteWithIdAndLocation(partitionIds[1], ResourceID.generate()), // Unknown
    new UnknownShuffleDescriptor(new ResultPartitionID(partitionIds[2], new ExecutionAttemptID())) };
    InputGateDeploymentDescriptor gateDesc = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), resultPartitionType, subpartitionIndexRange, new TaskDeploymentDescriptor.NonOffloaded<>(CompressedSerializedValue.fromObject(channelDescs)));
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    return new SingleInputGateFactory(localLocation, netEnv.getConfiguration(), connectionManager != null ? connectionManager : netEnv.getConnectionManager(), resultPartitionManager != null ? resultPartitionManager : netEnv.getResultPartitionManager(), new TaskEventDispatcher(), netEnv.getNetworkBufferPool()).create(netEnv.createShuffleIOOwnerContext("TestTask", taskMetricGroup.executionId(), taskMetricGroup), 0, gateDesc, SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER);
}