Examples with AllocationID org.apache.flink.runtime.clusterframework.types.AllocationID used on opensource projects

Example 56 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedSlotManagerTest method testTimeoutForUnusedTaskManager.

// ---------------------------------------------------------------------------------------------
// Task manager timeout
// ---------------------------------------------------------------------------------------------
/**
 * Tests that formerly used task managers can timeout after all of their slots have been freed.
 */
@Test
public void testTimeoutForUnusedTaskManager() throws Exception {
    final Time taskManagerTimeout = Time.milliseconds(50L);
    final CompletableFuture<InstanceID> releaseResourceFuture = new CompletableFuture<>();
    final AllocationID allocationId = new AllocationID();
    final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection();
    final InstanceID instanceId = taskExecutionConnection.getInstanceID();
    new Context() {

        {
            resourceActionsBuilder.setReleaseResourceConsumer((instanceID, e) -> releaseResourceFuture.complete(instanceID));
            slotManagerConfigurationBuilder.setTaskManagerTimeout(taskManagerTimeout);
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture = new CompletableFuture<>();
                runInMainThread(() -> registerTaskManagerFuture.complete(getSlotManager().registerTaskManager(taskExecutionConnection, new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE)), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture), is(true));
                assertEquals(getSlotManager().getTaskManagerIdleSince(instanceId), Long.MAX_VALUE);
                final CompletableFuture<Long> idleSinceFuture = new CompletableFuture<>();
                runInMainThread(() -> {
                    getSlotManager().freeSlot(new SlotID(taskExecutionConnection.getResourceID(), 0), allocationId);
                    idleSinceFuture.complete(getSlotManager().getTaskManagerIdleSince(instanceId));
                });
                assertThat(assertFutureCompleteAndReturn(idleSinceFuture), not(equalTo(Long.MAX_VALUE)));
                assertThat(assertFutureCompleteAndReturn(releaseResourceFuture), is(equalTo(instanceId)));
                // A task manager timeout does not remove the slots from the
                // SlotManager. The receiver of the callback can then decide what to do
                // with the TaskManager.
                assertEquals(DEFAULT_NUM_SLOTS_PER_WORKER, getSlotManager().getNumberRegisteredSlots());
                final CompletableFuture<Boolean> unregisterTaskManagerFuture = new CompletableFuture<>();
                runInMainThread(() -> unregisterTaskManagerFuture.complete(getSlotManager().unregisterTaskManager(taskExecutionConnection.getInstanceID(), TEST_EXCEPTION)));
                assertThat(assertFutureCompleteAndReturn(unregisterTaskManagerFuture), is(true));
                assertEquals(0, getSlotManager().getNumberRegisteredSlots());
            });
        }
    };
}

Example 57 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedSlotManagerTest method testGetResourceOverview.

@Test
public void testGetResourceOverview() throws Exception {
    final TaskExecutorConnection taskExecutorConnection1 = createTaskExecutorConnection();
    final TaskExecutorConnection taskExecutorConnection2 = createTaskExecutorConnection();
    final ResourceID resourceId1 = ResourceID.generate();
    final ResourceID resourceId2 = ResourceID.generate();
    final SlotID slotId1 = new SlotID(resourceId1, 0);
    final SlotID slotId2 = new SlotID(resourceId2, 0);
    final ResourceProfile resourceProfile1 = ResourceProfile.fromResources(1, 10);
    final ResourceProfile resourceProfile2 = ResourceProfile.fromResources(2, 20);
    final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile1, new JobID(), new AllocationID());
    final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile2, new JobID(), new AllocationID());
    final SlotReport slotReport1 = new SlotReport(slotStatus1);
    final SlotReport slotReport2 = new SlotReport(slotStatus2);
    new Context() {

        {
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture1 = new CompletableFuture<>();
                final CompletableFuture<Boolean> registerTaskManagerFuture2 = new CompletableFuture<>();
                runInMainThread(() -> {
                    registerTaskManagerFuture1.complete(getSlotManager().registerTaskManager(taskExecutorConnection1, slotReport1, resourceProfile1.multiply(2), resourceProfile1));
                    registerTaskManagerFuture2.complete(getSlotManager().registerTaskManager(taskExecutorConnection2, slotReport2, resourceProfile2.multiply(2), resourceProfile2));
                });
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture1), is(true));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture2), is(true));
                assertThat(getSlotManager().getFreeResource(), equalTo(resourceProfile1.merge(resourceProfile2)));
                assertThat(getSlotManager().getFreeResourceOf(taskExecutorConnection1.getInstanceID()), equalTo(resourceProfile1));
                assertThat(getSlotManager().getFreeResourceOf(taskExecutorConnection2.getInstanceID()), equalTo(resourceProfile2));
                assertThat(getSlotManager().getRegisteredResource(), equalTo(resourceProfile1.merge(resourceProfile2).multiply(2)));
                assertThat(getSlotManager().getRegisteredResourceOf(taskExecutorConnection1.getInstanceID()), equalTo(resourceProfile1.multiply(2)));
                assertThat(getSlotManager().getRegisteredResourceOf(taskExecutorConnection2.getInstanceID()), equalTo(resourceProfile2.multiply(2)));
            });
        }
    };
}

Example 58 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all allocated slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final AllocationID allocationId = new AllocationID();
    final SlotReport slotReport = new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE));
    new Context() {

        {
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture = new CompletableFuture<>();
                final CompletableFuture<Boolean> unRegisterTaskManagerFuture = new CompletableFuture<>();
                runInMainThread(() -> registerTaskManagerFuture.complete(getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers().size(), is(1));
                final Optional<TaskManagerSlotInformation> slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId);
                assertTrue(slot.isPresent());
                assertTrue(slot.get().getState() == SlotState.ALLOCATED);
                runInMainThread(() -> unRegisterTaskManagerFuture.complete(getSlotManager().unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION)));
                assertThat(assertFutureCompleteAndReturn(unRegisterTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers(), is(empty()));
                assertFalse(getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId).isPresent());
            });
        }
    };
}

Example 59 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedTaskManagerTrackerTest method testFreeSlot.

@Test
public void testFreeSlot() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final ResourceProfile totalResource = ResourceProfile.fromResources(10, 1000);
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final JobID jobId = new JobID();
    taskManagerTracker.addTaskManager(TASK_EXECUTOR_CONNECTION, totalResource, totalResource);
    taskManagerTracker.notifySlotStatus(allocationId1, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), ResourceProfile.fromResources(3, 200), SlotState.PENDING);
    taskManagerTracker.notifySlotStatus(allocationId2, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), ResourceProfile.fromResources(2, 300), SlotState.ALLOCATED);
    // Free pending slot
    taskManagerTracker.notifySlotStatus(allocationId1, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), ResourceProfile.fromResources(3, 200), SlotState.FREE);
    assertFalse(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertThat(taskManagerTracker.getRegisteredTaskManager(TASK_EXECUTOR_CONNECTION.getInstanceID()).get().getAvailableResource(), is(ResourceProfile.fromResources(8, 700)));
    // Free allocated slot
    taskManagerTracker.notifySlotStatus(allocationId2, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), ResourceProfile.fromResources(2, 300), SlotState.FREE);
    assertFalse(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    assertThat(taskManagerTracker.getRegisteredTaskManager(TASK_EXECUTOR_CONNECTION.getInstanceID()).get().getAvailableResource(), is(totalResource));
}

Example 60 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedTaskManagerTrackerTest method testGetStatistics.

@Test
public void testGetStatistics() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final ResourceProfile totalResource = ResourceProfile.fromResources(10, 1000);
    final ResourceProfile defaultSlotResource = ResourceProfile.fromResources(1, 100);
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final JobID jobId = new JobID();
    taskManagerTracker.addTaskManager(TASK_EXECUTOR_CONNECTION, totalResource, defaultSlotResource);
    taskManagerTracker.notifySlotStatus(allocationId1, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), ResourceProfile.fromResources(3, 200), SlotState.ALLOCATED);
    taskManagerTracker.notifySlotStatus(allocationId2, jobId, TASK_EXECUTOR_CONNECTION.getInstanceID(), defaultSlotResource, SlotState.ALLOCATED);
    taskManagerTracker.addPendingTaskManager(new PendingTaskManager(ResourceProfile.fromResources(4, 200), 1));
    assertThat(taskManagerTracker.getFreeResource(), is(ResourceProfile.fromResources(6, 700)));
    assertThat(taskManagerTracker.getRegisteredResource(), is(totalResource));
    assertThat(taskManagerTracker.getNumberRegisteredSlots(), is(10));
    assertThat(taskManagerTracker.getNumberFreeSlots(), is(8));
    assertThat(taskManagerTracker.getPendingResource(), is(ResourceProfile.fromResources(4, 200)));
}