Examples with ShuffleDescriptor org.apache.flink.runtime.shuffle.ShuffleDescriptor used on opensource projects

Example 1 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method testMixedLocalRemoteUnknownDeployment.

/**
 * Tests the deployment descriptors for local, remote, and unknown partition locations (with
 * lazy deployment allowed and all execution states for the producers).
 */
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
    ResourceID consumerResourceID = ResourceID.generate();
    JobID jobID = new JobID();
    // states.
    for (ExecutionState state : ExecutionState.values()) {
        ResultPartitionID localPartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor localPartition = createResultPartitionDeploymentDescriptor(jobID, localPartitionId, consumerResourceID);
        ResultPartitionID remotePartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor remotePartition = createResultPartitionDeploymentDescriptor(jobID, remotePartitionId, ResourceID.generate());
        ResultPartitionID unknownPartitionId = new ResultPartitionID();
        ShuffleDescriptor localShuffleDescriptor = getConsumedPartitionShuffleDescriptor(localPartitionId, state, localPartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor remoteShuffleDescriptor = getConsumedPartitionShuffleDescriptor(remotePartitionId, state, remotePartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor unknownShuffleDescriptor = getConsumedPartitionShuffleDescriptor(unknownPartitionId, state, null, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        // These states are allowed
        if (state == ExecutionState.RUNNING || state == ExecutionState.INITIALIZING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
            NettyShuffleDescriptor nettyShuffleDescriptor;
            // Create local or remote channels
            verifyShuffleDescriptor(localShuffleDescriptor, NettyShuffleDescriptor.class, false, localPartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) localShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(true));
            verifyShuffleDescriptor(remoteShuffleDescriptor, NettyShuffleDescriptor.class, false, remotePartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) remoteShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(false));
            assertThat(nettyShuffleDescriptor.getConnectionId(), is(STUB_CONNECTION_ID));
        } else {
            // Unknown (lazy deployment allowed)
            verifyShuffleDescriptor(localShuffleDescriptor, UnknownShuffleDescriptor.class, true, localPartitionId);
            verifyShuffleDescriptor(remoteShuffleDescriptor, UnknownShuffleDescriptor.class, true, remotePartitionId);
        }
        verifyShuffleDescriptor(unknownShuffleDescriptor, UnknownShuffleDescriptor.class, true, unknownPartitionId);
    }
}

Example 2 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method testUnknownDescriptorWithOrWithoutLazyDeployment.

@Test
public void testUnknownDescriptorWithOrWithoutLazyDeployment() {
    ResultPartitionID unknownPartitionId = new ResultPartitionID();
    // This should work if lazy deployment is allowed
    ShuffleDescriptor unknownSdd = getConsumedPartitionShuffleDescriptor(unknownPartitionId, ExecutionState.CREATED, null, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
    assertThat(unknownSdd, instanceOf(UnknownShuffleDescriptor.class));
    assertThat(unknownSdd.isUnknown(), is(true));
    assertThat(unknownSdd.getResultPartitionID(), is(unknownPartitionId));
    try {
        // Fail if lazy deployment is *not* allowed
        getConsumedPartitionShuffleDescriptor(unknownPartitionId, ExecutionState.CREATED, null, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.MUST_BE_KNOWN);
        fail("Did not throw expected ExecutionGraphException");
    } catch (IllegalStateException ignored) {
    }
}

Example 3 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class ResultPartitionDeploymentDescriptorTest method testSerializationOfUnknownShuffleDescriptor.

/**
 * Tests simple de/serialization with {@link UnknownShuffleDescriptor}.
 */
@Test
public void testSerializationOfUnknownShuffleDescriptor() throws IOException {
    ShuffleDescriptor shuffleDescriptor = new UnknownShuffleDescriptor(resultPartitionID);
    ShuffleDescriptor shuffleDescriptorCopy = CommonTestUtils.createCopySerializable(shuffleDescriptor);
    assertThat(shuffleDescriptorCopy, instanceOf(UnknownShuffleDescriptor.class));
    assertThat(shuffleDescriptorCopy.getResultPartitionID(), is(resultPartitionID));
    assertThat(shuffleDescriptorCopy.isUnknown(), is(true));
}

Example 4 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class Execution method createPartitionInfo.

private static PartitionInfo createPartitionInfo(IntermediateResultPartition consumedPartition) {
    IntermediateDataSetID intermediateDataSetID = consumedPartition.getIntermediateResult().getId();
    ShuffleDescriptor shuffleDescriptor = getConsumedPartitionShuffleDescriptor(consumedPartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.MUST_BE_KNOWN);
    return new PartitionInfo(intermediateDataSetID, shuffleDescriptor);
}

Example 5 with ShuffleDescriptor

use of org.apache.flink.runtime.shuffle.ShuffleDescriptor in project flink by apache.

the class InputGateDeploymentDescriptor method loadBigData.

public void loadBigData(@Nullable PermanentBlobService blobService, JobID jobId) throws IOException, ClassNotFoundException {
    if (serializedInputChannels instanceof Offloaded) {
        PermanentBlobKey blobKey = ((Offloaded<ShuffleDescriptor[]>) serializedInputChannels).serializedValueKey;
        Preconditions.checkNotNull(blobService);
        // NOTE: Do not delete the ShuffleDescriptor BLOBs since it may be needed again during
        // recovery. (it is deleted automatically on the BLOB server and cache when its
        // partition is no longer available or the job enters a terminal state)
        CompressedSerializedValue<ShuffleDescriptor[]> serializedValue = CompressedSerializedValue.fromBytes(blobService.readFile(jobId, blobKey));
        serializedInputChannels = new NonOffloaded<>(serializedValue);
        Preconditions.checkNotNull(serializedInputChannels);
    }
}