Examples with NettyShuffleDescriptor org.apache.flink.runtime.shuffle.NettyShuffleDescriptor used on opensource projects

Example 1 with NettyShuffleDescriptor

use of org.apache.flink.runtime.shuffle.NettyShuffleDescriptor in project flink by apache.

the class ShuffleDescriptorTest method testMixedLocalRemoteUnknownDeployment.

/**
 * Tests the deployment descriptors for local, remote, and unknown partition locations (with
 * lazy deployment allowed and all execution states for the producers).
 */
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
    ResourceID consumerResourceID = ResourceID.generate();
    JobID jobID = new JobID();
    // states.
    for (ExecutionState state : ExecutionState.values()) {
        ResultPartitionID localPartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor localPartition = createResultPartitionDeploymentDescriptor(jobID, localPartitionId, consumerResourceID);
        ResultPartitionID remotePartitionId = new ResultPartitionID();
        ResultPartitionDeploymentDescriptor remotePartition = createResultPartitionDeploymentDescriptor(jobID, remotePartitionId, ResourceID.generate());
        ResultPartitionID unknownPartitionId = new ResultPartitionID();
        ShuffleDescriptor localShuffleDescriptor = getConsumedPartitionShuffleDescriptor(localPartitionId, state, localPartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor remoteShuffleDescriptor = getConsumedPartitionShuffleDescriptor(remotePartitionId, state, remotePartition, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        ShuffleDescriptor unknownShuffleDescriptor = getConsumedPartitionShuffleDescriptor(unknownPartitionId, state, null, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.CAN_BE_UNKNOWN);
        // These states are allowed
        if (state == ExecutionState.RUNNING || state == ExecutionState.INITIALIZING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
            NettyShuffleDescriptor nettyShuffleDescriptor;
            // Create local or remote channels
            verifyShuffleDescriptor(localShuffleDescriptor, NettyShuffleDescriptor.class, false, localPartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) localShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(true));
            verifyShuffleDescriptor(remoteShuffleDescriptor, NettyShuffleDescriptor.class, false, remotePartitionId);
            nettyShuffleDescriptor = (NettyShuffleDescriptor) remoteShuffleDescriptor;
            assertThat(nettyShuffleDescriptor.isLocalTo(consumerResourceID), is(false));
            assertThat(nettyShuffleDescriptor.getConnectionId(), is(STUB_CONNECTION_ID));
        } else {
            // Unknown (lazy deployment allowed)
            verifyShuffleDescriptor(localShuffleDescriptor, UnknownShuffleDescriptor.class, true, localPartitionId);
            verifyShuffleDescriptor(remoteShuffleDescriptor, UnknownShuffleDescriptor.class, true, remotePartitionId);
        }
        verifyShuffleDescriptor(unknownShuffleDescriptor, UnknownShuffleDescriptor.class, true, unknownPartitionId);
    }
}

Example 2 with NettyShuffleDescriptor

use of org.apache.flink.runtime.shuffle.NettyShuffleDescriptor in project flink by apache.

the class NettyShuffleEnvironment method updatePartitionInfo.

@Override
public boolean updatePartitionInfo(ExecutionAttemptID consumerID, PartitionInfo partitionInfo) throws IOException, InterruptedException {
    IntermediateDataSetID intermediateResultPartitionID = partitionInfo.getIntermediateDataSetID();
    InputGateID id = new InputGateID(intermediateResultPartitionID, consumerID);
    SingleInputGate inputGate = inputGatesById.get(id);
    if (inputGate == null) {
        return false;
    }
    ShuffleDescriptor shuffleDescriptor = partitionInfo.getShuffleDescriptor();
    checkArgument(shuffleDescriptor instanceof NettyShuffleDescriptor, "Tried to update unknown channel with unknown ShuffleDescriptor %s.", shuffleDescriptor.getClass().getName());
    inputGate.updateInputChannel(taskExecutorResourceId, (NettyShuffleDescriptor) shuffleDescriptor);
    return true;
}

Example 3 with NettyShuffleDescriptor

use of org.apache.flink.runtime.shuffle.NettyShuffleDescriptor in project flink by apache.

the class TaskExecutorSubmissionTest method testFailingNotifyPartitionDataAvailable.

/**
 * Test that a failing notifyPartitionDataAvailable call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploying consumer tasks and
 * requiring the invokable to fill one memory segment. The completed memory segment will trigger
 * the scheduling of the downstream operator since it is in pipeline mode. After we've filled
 * the memory segment, we'll block the invokable and wait for the task failure due to the failed
 * notifyPartitionDataAvailable call.
 */
@Test
public void testFailingNotifyPartitionDataAvailable() throws Exception {
    final Configuration configuration = new Configuration();
    // set the memory segment to the smallest size possible, because we have to fill one
    // memory buffer to trigger notifyPartitionDataAvailable to the downstream
    // operators
    configuration.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
    NettyShuffleDescriptor sdd = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
    TaskDeploymentDescriptor tdd = createSender(sdd, TestingAbstractInvokables.TestInvokableRecordCancel.class);
    ExecutionAttemptID eid = tdd.getExecutionAttemptId();
    final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
    final Exception exception = new Exception("Failed notifyPartitionDataAvailable");
    final JobMasterId jobMasterId = JobMasterId.generate();
    TestingJobMasterGateway testingJobMasterGateway = new TestingJobMasterGatewayBuilder().setFencingTokenSupplier(() -> jobMasterId).setNotifyPartitionDataAvailableFunction(resultPartitionID -> FutureUtils.completedExceptionally(exception)).build();
    try (TaskSubmissionTestEnvironment env = new TaskSubmissionTestEnvironment.Builder(jobId).setSlotSize(1).setConfiguration(configuration).addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture).setJobMasterId(jobMasterId).setJobMasterGateway(testingJobMasterGateway).useRealNonMockShuffleEnvironment().build()) {
        TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
        TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();
        TestingAbstractInvokables.TestInvokableRecordCancel.resetGotCanceledFuture();
        taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd, jobMasterId, timeout).get();
        taskRunningFuture.get();
        CompletableFuture<Boolean> cancelFuture = TestingAbstractInvokables.TestInvokableRecordCancel.gotCanceled();
        assertTrue(cancelFuture.get());
        assertTrue(ExceptionUtils.findThrowableWithMessage(taskSlotTable.getTask(eid).getFailureCause(), exception.getMessage()).isPresent());
    }
}

Example 4 with NettyShuffleDescriptor

use of org.apache.flink.runtime.shuffle.NettyShuffleDescriptor in project flink by apache.

the class TaskExecutorSubmissionTest method testRunJobWithForwardChannel.

@Test
public void testRunJobWithForwardChannel() throws Exception {
    ResourceID producerLocation = ResourceID.generate();
    NettyShuffleDescriptor sdd = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), producerLocation);
    TaskDeploymentDescriptor tdd1 = createSender(sdd);
    TaskDeploymentDescriptor tdd2 = createReceiver(sdd);
    ExecutionAttemptID eid1 = tdd1.getExecutionAttemptId();
    ExecutionAttemptID eid2 = tdd2.getExecutionAttemptId();
    final CompletableFuture<Void> task1RunningFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task2RunningFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task1FinishedFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task2FinishedFuture = new CompletableFuture<>();
    final JobMasterId jobMasterId = JobMasterId.generate();
    TestingJobMasterGateway testingJobMasterGateway = new TestingJobMasterGatewayBuilder().setFencingTokenSupplier(() -> jobMasterId).setNotifyPartitionDataAvailableFunction(resultPartitionID -> CompletableFuture.completedFuture(Acknowledge.get())).build();
    try (TaskSubmissionTestEnvironment env = new TaskSubmissionTestEnvironment.Builder(jobId).setResourceID(producerLocation).setSlotSize(2).addTaskManagerActionListener(eid1, ExecutionState.RUNNING, task1RunningFuture).addTaskManagerActionListener(eid2, ExecutionState.RUNNING, task2RunningFuture).addTaskManagerActionListener(eid1, ExecutionState.FINISHED, task1FinishedFuture).addTaskManagerActionListener(eid2, ExecutionState.FINISHED, task2FinishedFuture).setJobMasterId(jobMasterId).setJobMasterGateway(testingJobMasterGateway).useRealNonMockShuffleEnvironment().build()) {
        TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
        TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();
        taskSlotTable.allocateSlot(0, jobId, tdd1.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd1, jobMasterId, timeout).get();
        task1RunningFuture.get();
        taskSlotTable.allocateSlot(1, jobId, tdd2.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd2, jobMasterId, timeout).get();
        task2RunningFuture.get();
        task1FinishedFuture.get();
        task2FinishedFuture.get();
        assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.FINISHED);
        assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.FINISHED);
    }
}

Example 5 with NettyShuffleDescriptor

use of org.apache.flink.runtime.shuffle.NettyShuffleDescriptor in project flink by apache.

the class ResultPartitionDeploymentDescriptorTest method testSerializationWithNettyShuffleDescriptor.

/**
 * Tests simple de/serialization with {@link NettyShuffleDescriptor}.
 */
@Test
public void testSerializationWithNettyShuffleDescriptor() throws IOException {
    ShuffleDescriptor shuffleDescriptor = new NettyShuffleDescriptor(producerLocation, new NetworkPartitionConnectionInfo(connectionID), resultPartitionID);
    ResultPartitionDeploymentDescriptor copy = createCopyAndVerifyResultPartitionDeploymentDescriptor(shuffleDescriptor);
    assertThat(copy.getShuffleDescriptor(), instanceOf(NettyShuffleDescriptor.class));
    NettyShuffleDescriptor shuffleDescriptorCopy = (NettyShuffleDescriptor) copy.getShuffleDescriptor();
    assertThat(shuffleDescriptorCopy.getResultPartitionID(), is(resultPartitionID));
    assertThat(shuffleDescriptorCopy.isUnknown(), is(false));
    assertThat(shuffleDescriptorCopy.isLocalTo(producerLocation), is(true));
    assertThat(shuffleDescriptorCopy.getConnectionId(), is(connectionID));
}