Examples with SimpleAckingTaskManagerGateway org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway used on opensource projects

Example 11 with SimpleAckingTaskManagerGateway

use of org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway in project flink by apache.

the class DeclarativeSlotPoolBridgeRequestCompletionTest method runSlotRequestCompletionTest.

private void runSlotRequestCompletionTest(Supplier<SlotPool> slotPoolSupplier, Consumer<SlotPool> actionAfterSlotRequest) {
    try (final SlotPool slotPool = slotPoolSupplier.get()) {
        final int requestNum = 10;
        final List<SlotRequestId> slotRequestIds = IntStream.range(0, requestNum).mapToObj(ignored -> new SlotRequestId()).collect(Collectors.toList());
        final List<CompletableFuture<PhysicalSlot>> slotRequests = slotRequestIds.stream().map(slotRequestId -> slotPool.requestNewAllocatedSlot(slotRequestId, ResourceProfile.UNKNOWN, TIMEOUT)).collect(Collectors.toList());
        actionAfterSlotRequest.accept(slotPool);
        final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
        slotPool.registerTaskManager(taskManagerLocation.getResourceID());
        // create a slot offer that is initiated by the last request
        final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);
        final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(slotOffer));
        assertThat(acceptedSlots, containsInAnyOrder(slotOffer));
        final FlinkException testingReleaseException = new FlinkException("Testing release exception");
        // check that the slot requests get completed in sequential order
        for (int i = 0; i < slotRequestIds.size(); i++) {
            final CompletableFuture<PhysicalSlot> slotRequestFuture = slotRequests.get(i);
            assertThat(slotRequestFuture.getNow(null), is(not(nullValue())));
            slotPool.releaseSlot(slotRequestIds.get(i), testingReleaseException);
        }
    }
}

Example 12 with SimpleAckingTaskManagerGateway

use of org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway in project flink by apache.

the class DeclarativeSlotPoolBridgeTest method testAcceptingOfferedSlotsWithoutResourceManagerConnected.

@Test
public void testAcceptingOfferedSlotsWithoutResourceManagerConnected() throws Exception {
    try (DeclarativeSlotPoolBridge declarativeSlotPoolBridge = createDeclarativeSlotPoolBridge(new DefaultDeclarativeSlotPoolFactory(), requestSlotMatchingStrategy)) {
        declarativeSlotPoolBridge.start(jobMasterId, "localhost", mainThreadExecutor);
        final CompletableFuture<PhysicalSlot> slotFuture = declarativeSlotPoolBridge.requestNewAllocatedSlot(new SlotRequestId(), ResourceProfile.UNKNOWN, rpcTimeout);
        final LocalTaskManagerLocation localTaskManagerLocation = new LocalTaskManagerLocation();
        declarativeSlotPoolBridge.registerTaskManager(localTaskManagerLocation.getResourceID());
        final AllocationID allocationId = new AllocationID();
        declarativeSlotPoolBridge.offerSlots(localTaskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(new SlotOffer(allocationId, 0, ResourceProfile.ANY)));
        assertThat(slotFuture.join().getAllocationId(), is(allocationId));
    }
}

Example 13 with SimpleAckingTaskManagerGateway

use of org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway in project flink by apache.

the class DeclarativeSlotPoolBridgePreferredAllocationsTest method testDeclarativeSlotPoolTakesPreferredAllocationsIntoAccount.

@Test
public void testDeclarativeSlotPoolTakesPreferredAllocationsIntoAccount() throws Exception {
    final DeclarativeSlotPoolBridge declarativeSlotPoolBridge = new DeclarativeSlotPoolBridge(new JobID(), new DefaultDeclarativeSlotPoolFactory(), SystemClock.getInstance(), TestingUtils.infiniteTime(), TestingUtils.infiniteTime(), TestingUtils.infiniteTime(), PreferredAllocationRequestSlotMatchingStrategy.INSTANCE);
    declarativeSlotPoolBridge.start(JobMasterId.generate(), "localhost", ComponentMainThreadExecutorServiceAdapter.forMainThread());
    final LocalTaskManagerLocation localTaskManagerLocation = new LocalTaskManagerLocation();
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final CompletableFuture<PhysicalSlot> slotRequestWithPreferredAllocation1 = requestSlot(declarativeSlotPoolBridge, Collections.singleton(allocationId1));
    final CompletableFuture<PhysicalSlot> slotRequestWithEmptyPreferredAllocations = requestSlot(declarativeSlotPoolBridge, Collections.emptySet());
    final CompletableFuture<PhysicalSlot> slotRequestWithPreferredAllocation2 = requestSlot(declarativeSlotPoolBridge, Collections.singleton(allocationId2));
    final Collection<SlotOffer> slotOffers = new ArrayList<>();
    slotOffers.add(new SlotOffer(allocationId2, 0, ResourceProfile.ANY));
    final AllocationID otherAllocationId = new AllocationID();
    slotOffers.add(new SlotOffer(otherAllocationId, 1, ResourceProfile.ANY));
    slotOffers.add(new SlotOffer(allocationId1, 2, ResourceProfile.ANY));
    declarativeSlotPoolBridge.registerTaskManager(localTaskManagerLocation.getResourceID());
    declarativeSlotPoolBridge.offerSlots(localTaskManagerLocation, new SimpleAckingTaskManagerGateway(), slotOffers);
    assertThat(slotRequestWithPreferredAllocation1.join().getAllocationId()).isEqualTo(allocationId1);
    assertThat(slotRequestWithPreferredAllocation2.join().getAllocationId()).isEqualTo(allocationId2);
    assertThat(slotRequestWithEmptyPreferredAllocations.join().getAllocationId()).isEqualTo(otherAllocationId);
}

Example 14 with SimpleAckingTaskManagerGateway

use of org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway in project flink by apache.

the class CheckpointCoordinatorTest method testReportLatestCompletedCheckpointIdWithAbort.

@Test
public void testReportLatestCompletedCheckpointIdWithAbort() throws Exception {
    JobVertexID jobVertexID = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID).setTransitToRunning(false).build();
    ExecutionVertex task = graph.getJobVertex(jobVertexID).getTaskVertices()[0];
    AtomicLong reportedCheckpointId = new AtomicLong(-1);
    LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(new SimpleAckingTaskManagerGateway() {

        @Override
        public void notifyCheckpointAborted(ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long latestCompletedCheckpointId, long timestamp) {
            reportedCheckpointId.set(latestCompletedCheckpointId);
        }
    }).createTestingLogicalSlot();
    ExecutionGraphTestUtils.setVertexResource(task, slot);
    task.getCurrentExecutionAttempt().transitionState(ExecutionState.RUNNING);
    CheckpointCoordinator checkpointCoordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setTimer(manuallyTriggeredScheduledExecutor).setAllowCheckpointsAfterTasksFinished(true).build();
    // Trigger a successful checkpoint
    CompletableFuture<CompletedCheckpoint> result = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    long completedCheckpointId = checkpointCoordinator.getPendingCheckpoints().entrySet().iterator().next().getKey();
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), task.getCurrentExecutionAttempt().getAttemptId(), completedCheckpointId, new CheckpointMetrics(), new TaskStateSnapshot()), "localhost");
    assertTrue(result.isDone());
    assertFalse(result.isCompletedExceptionally());
    result = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    long abortedCheckpointId = checkpointCoordinator.getPendingCheckpoints().entrySet().iterator().next().getKey();
    checkpointCoordinator.receiveDeclineMessage(new DeclineCheckpoint(graph.getJobID(), task.getCurrentExecutionAttempt().getAttemptId(), abortedCheckpointId, new CheckpointException(CHECKPOINT_EXPIRED)), "localhost");
    assertTrue(result.isCompletedExceptionally());
    assertEquals(completedCheckpointId, reportedCheckpointId.get());
}