Examples with TaskExecutorConnection org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection used on opensource projects

Example 16 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class FineGrainedSlotManagerTest method testTaskManagerUnregistration.

/**
 * Tests that un-registration of task managers will free and remove all allocated slots.
 */
@Test
public void testTaskManagerUnregistration() throws Exception {
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final AllocationID allocationId = new AllocationID();
    final SlotReport slotReport = new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE));
    new Context() {

        {
            runTest(() -> {
                final CompletableFuture<Boolean> registerTaskManagerFuture = new CompletableFuture<>();
                final CompletableFuture<Boolean> unRegisterTaskManagerFuture = new CompletableFuture<>();
                runInMainThread(() -> registerTaskManagerFuture.complete(getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE)));
                assertThat(assertFutureCompleteAndReturn(registerTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers().size(), is(1));
                final Optional<TaskManagerSlotInformation> slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId);
                assertTrue(slot.isPresent());
                assertTrue(slot.get().getState() == SlotState.ALLOCATED);
                runInMainThread(() -> unRegisterTaskManagerFuture.complete(getSlotManager().unregisterTaskManager(taskManagerConnection.getInstanceID(), TEST_EXCEPTION)));
                assertThat(assertFutureCompleteAndReturn(unRegisterTaskManagerFuture), is(true));
                assertThat(getTaskManagerTracker().getRegisteredTaskManagers(), is(empty()));
                assertFalse(getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId).isPresent());
            });
        }
    };
}

Example 17 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class TaskExecutorManagerTest method testPendingSlotNotFulfilledByAllocatedSlot.

/**
 * Tests that a pending slot is not fulfilled by an already allocated slot.
 */
@Test
public void testPendingSlotNotFulfilledByAllocatedSlot() {
    final int numWorkerCpuCores = 3;
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(numWorkerCpuCores).build();
    final ResourceProfile requestedSlotProfile = ResourceProfile.newBuilder().setCpuCores(numWorkerCpuCores).build();
    try (final TaskExecutorManager taskExecutorManager = createTaskExecutorManagerBuilder().setDefaultWorkerResourceSpec(workerResourceSpec).setNumSlotsPerWorker(// set to one so that the slot profiles directly correspond to
    1).setMaxNumSlots(2).createTaskExecutorManager()) {
        // create pending slot
        taskExecutorManager.allocateWorker(requestedSlotProfile);
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
        final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection();
        final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorConnection.getResourceID(), 0), requestedSlotProfile, JobID.generate(), new AllocationID()));
        taskExecutorManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        // the slot from the task executor should be accepted, but we should still be waiting
        // for the originally requested slot
        assertThat(taskExecutorManager.getNumberRegisteredSlots(), is(1));
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
    }
}

Example 18 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DefaultSlotStatusSyncerTest method testSlotStatusProcessing.

@Test
public void testSlotStatusProcessing() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final JobID jobId = new JobID();
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final SlotID slotId1 = new SlotID(taskExecutorConnection.getResourceID(), 0);
    final SlotID slotId2 = new SlotID(taskExecutorConnection.getResourceID(), 1);
    final SlotID slotId3 = new SlotID(taskExecutorConnection.getResourceID(), 2);
    final ResourceProfile totalResource = ResourceProfile.fromResources(5, 20);
    final ResourceProfile resource = ResourceProfile.fromResources(1, 4);
    final SlotReport slotReport1 = new SlotReport(Arrays.asList(new SlotStatus(slotId1, totalResource), new SlotStatus(slotId2, resource, jobId, allocationId1), new SlotStatus(slotId3, resource, jobId, allocationId2)));
    final SlotReport slotReport2 = new SlotReport(Arrays.asList(new SlotStatus(slotId3, resource), new SlotStatus(slotId2, resource, jobId, allocationId1)));
    taskManagerTracker.addTaskManager(taskExecutorConnection, totalResource, totalResource);
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport1);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", resource);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 3)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(2, 8)));
    final AllocationID allocationId3 = taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAllocatedSlots().keySet().stream().filter(allocationId -> !allocationId.equals(allocationId1) && !allocationId.equals(allocationId2)).findAny().get();
    // allocationId1 should still be allocated; allocationId2 should be freed; allocationId3
    // should continue to be in a pending state;
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport2);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertFalse(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).isPresent());
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).get().getState(), is(SlotState.ALLOCATED));
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).get().getState(), is(SlotState.PENDING));
}

Example 19 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DefaultSlotStatusSyncerTest method testAllocateSlotFailsWithException.

@Test
public void testAllocateSlotFailsWithException() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> FutureUtils.completedExceptionally(new TimeoutException("timeout"))).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    taskManagerTracker.addTaskManager(taskExecutorConnection, ResourceProfile.ANY, ResourceProfile.ANY);
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final JobID jobId = new JobID();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final CompletableFuture<Void> allocatedFuture = slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", ResourceProfile.ANY);
    try {
        allocatedFuture.get();
    } catch (Exception e) {
        assertThat(e.getCause(), instanceOf(TimeoutException.class));
    }
    assertThat(resourceTracker.getAcquiredResources(jobId), is(empty()));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAllocatedSlots().keySet(), is(empty()));
}

Example 20 with TaskExecutorConnection

use of org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection in project flink by apache.

the class DefaultSlotStatusSyncerTest method testAllocateSlot.

@Test
public void testAllocateSlot() throws Exception {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> responseFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(tuple6);
        return responseFuture;
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    taskManagerTracker.addTaskManager(taskExecutorConnection, ResourceProfile.ANY, ResourceProfile.ANY);
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final JobID jobId = new JobID();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final CompletableFuture<Void> allocatedFuture = slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", ResourceProfile.ANY);
    final AllocationID allocationId = requestFuture.get().f2;
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(ResourceProfile.ANY, 1)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).isPresent());
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).get().getJobId(), is(jobId));
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).get().getState(), is(SlotState.PENDING));
    responseFuture.complete(Acknowledge.get());
    assertFalse(allocatedFuture.isCompletedExceptionally());
}