Examples with IntermediateResultPartitionID org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID used on opensource projects

Example 21 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class PointwisePatternTest method test2NToN.

@Test
public void test2NToN() throws Exception {
    final int N = 17;
    ExecutionJobVertex target = setUpExecutionGraphAndGetDownstreamVertex(2 * N, N);
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ConsumedPartitionGroup consumedPartitionGroup = ev.getConsumedPartitionGroup(0);
        assertEquals(2, consumedPartitionGroup.size());
        int idx = 0;
        for (IntermediateResultPartitionID partitionId : consumedPartitionGroup) {
            assertEquals(ev.getParallelSubtaskIndex() * 2L + idx++, partitionId.getPartitionNumber());
        }
    }
}

Example 22 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class TaskExecutorSubmissionTest method testFailingNotifyPartitionDataAvailable.

/**
 * Test that a failing notifyPartitionDataAvailable call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploying consumer tasks and
 * requiring the invokable to fill one memory segment. The completed memory segment will trigger
 * the scheduling of the downstream operator since it is in pipeline mode. After we've filled
 * the memory segment, we'll block the invokable and wait for the task failure due to the failed
 * notifyPartitionDataAvailable call.
 */
@Test
public void testFailingNotifyPartitionDataAvailable() throws Exception {
    final Configuration configuration = new Configuration();
    // set the memory segment to the smallest size possible, because we have to fill one
    // memory buffer to trigger notifyPartitionDataAvailable to the downstream
    // operators
    configuration.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
    NettyShuffleDescriptor sdd = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
    TaskDeploymentDescriptor tdd = createSender(sdd, TestingAbstractInvokables.TestInvokableRecordCancel.class);
    ExecutionAttemptID eid = tdd.getExecutionAttemptId();
    final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
    final Exception exception = new Exception("Failed notifyPartitionDataAvailable");
    final JobMasterId jobMasterId = JobMasterId.generate();
    TestingJobMasterGateway testingJobMasterGateway = new TestingJobMasterGatewayBuilder().setFencingTokenSupplier(() -> jobMasterId).setNotifyPartitionDataAvailableFunction(resultPartitionID -> FutureUtils.completedExceptionally(exception)).build();
    try (TaskSubmissionTestEnvironment env = new TaskSubmissionTestEnvironment.Builder(jobId).setSlotSize(1).setConfiguration(configuration).addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture).setJobMasterId(jobMasterId).setJobMasterGateway(testingJobMasterGateway).useRealNonMockShuffleEnvironment().build()) {
        TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
        TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();
        TestingAbstractInvokables.TestInvokableRecordCancel.resetGotCanceledFuture();
        taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd, jobMasterId, timeout).get();
        taskRunningFuture.get();
        CompletableFuture<Boolean> cancelFuture = TestingAbstractInvokables.TestInvokableRecordCancel.gotCanceled();
        assertTrue(cancelFuture.get());
        assertTrue(ExceptionUtils.findThrowableWithMessage(taskSlotTable.getTask(eid).getFailureCause(), exception.getMessage()).isPresent());
    }
}

Example 23 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class TaskExecutorSubmissionTest method testRunJobWithForwardChannel.

@Test
public void testRunJobWithForwardChannel() throws Exception {
    ResourceID producerLocation = ResourceID.generate();
    NettyShuffleDescriptor sdd = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), producerLocation);
    TaskDeploymentDescriptor tdd1 = createSender(sdd);
    TaskDeploymentDescriptor tdd2 = createReceiver(sdd);
    ExecutionAttemptID eid1 = tdd1.getExecutionAttemptId();
    ExecutionAttemptID eid2 = tdd2.getExecutionAttemptId();
    final CompletableFuture<Void> task1RunningFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task2RunningFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task1FinishedFuture = new CompletableFuture<>();
    final CompletableFuture<Void> task2FinishedFuture = new CompletableFuture<>();
    final JobMasterId jobMasterId = JobMasterId.generate();
    TestingJobMasterGateway testingJobMasterGateway = new TestingJobMasterGatewayBuilder().setFencingTokenSupplier(() -> jobMasterId).setNotifyPartitionDataAvailableFunction(resultPartitionID -> CompletableFuture.completedFuture(Acknowledge.get())).build();
    try (TaskSubmissionTestEnvironment env = new TaskSubmissionTestEnvironment.Builder(jobId).setResourceID(producerLocation).setSlotSize(2).addTaskManagerActionListener(eid1, ExecutionState.RUNNING, task1RunningFuture).addTaskManagerActionListener(eid2, ExecutionState.RUNNING, task2RunningFuture).addTaskManagerActionListener(eid1, ExecutionState.FINISHED, task1FinishedFuture).addTaskManagerActionListener(eid2, ExecutionState.FINISHED, task2FinishedFuture).setJobMasterId(jobMasterId).setJobMasterGateway(testingJobMasterGateway).useRealNonMockShuffleEnvironment().build()) {
        TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
        TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();
        taskSlotTable.allocateSlot(0, jobId, tdd1.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd1, jobMasterId, timeout).get();
        task1RunningFuture.get();
        taskSlotTable.allocateSlot(1, jobId, tdd2.getAllocationId(), Time.seconds(60));
        tmGateway.submitTask(tdd2, jobMasterId, timeout).get();
        task2RunningFuture.get();
        task1FinishedFuture.get();
        task2FinishedFuture.get();
        assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.FINISHED);
        assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.FINISHED);
    }
}

Example 24 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class RegionPartitionGroupReleaseStrategyTest method toggleVertexFinishedUnfinished.

@Test
public void toggleVertexFinishedUnfinished() {
    final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
    testingSchedulingTopology.connectAllToAll(producers, consumers).finish();
    final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();
    final ExecutionVertexID consumerVertex2 = consumers.get(1).getId();
    final RegionPartitionGroupReleaseStrategy regionPartitionGroupReleaseStrategy = new RegionPartitionGroupReleaseStrategy(testingSchedulingTopology);
    regionPartitionGroupReleaseStrategy.vertexFinished(consumerVertex1);
    regionPartitionGroupReleaseStrategy.vertexFinished(consumerVertex2);
    regionPartitionGroupReleaseStrategy.vertexUnfinished(consumerVertex2);
    final List<IntermediateResultPartitionID> partitionsToRelease = getReleasablePartitions(regionPartitionGroupReleaseStrategy, consumerVertex1);
    assertThat(partitionsToRelease, is(empty()));
}

Example 25 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class RegionPartitionGroupReleaseStrategyTest method updateStrategyOnTopologyUpdate.

@Test
public void updateStrategyOnTopologyUpdate() {
    final TestingSchedulingExecutionVertex ev1 = testingSchedulingTopology.newExecutionVertex();
    final RegionPartitionGroupReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionGroupReleaseStrategy(testingSchedulingTopology);
    regionPartitionReleaseStrategy.vertexFinished(ev1.getId());
    final TestingSchedulingExecutionVertex ev2 = testingSchedulingTopology.newExecutionVertex();
    testingSchedulingTopology.connect(ev1, ev2, ResultPartitionType.BLOCKING);
    regionPartitionReleaseStrategy.notifySchedulingTopologyUpdated(testingSchedulingTopology, Collections.singletonList(ev2.getId()));
    // this check ensures that existing region views are not affected
    assertThat(regionPartitionReleaseStrategy.isRegionOfVertexFinished(ev1.getId()), is(true));
    assertThat(regionPartitionReleaseStrategy.isRegionOfVertexFinished(ev2.getId()), is(false));
    List<IntermediateResultPartitionID> releasablePartitions = getReleasablePartitions(regionPartitionReleaseStrategy, ev2.getId());
    assertThat(regionPartitionReleaseStrategy.isRegionOfVertexFinished(ev2.getId()), is(true));
    assertThat(releasablePartitions, contains(ev1.getProducedResults().iterator().next().getId()));
}