Examples with AllocationID org.apache.flink.runtime.clusterframework.types.AllocationID used on opensource projects

Example 61 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class TaskExecutorManagerTest method testPendingSlotNotFulfilledByAllocatedSlot.

/**
 * Tests that a pending slot is not fulfilled by an already allocated slot.
 */
@Test
public void testPendingSlotNotFulfilledByAllocatedSlot() {
    final int numWorkerCpuCores = 3;
    final WorkerResourceSpec workerResourceSpec = new WorkerResourceSpec.Builder().setCpuCores(numWorkerCpuCores).build();
    final ResourceProfile requestedSlotProfile = ResourceProfile.newBuilder().setCpuCores(numWorkerCpuCores).build();
    try (final TaskExecutorManager taskExecutorManager = createTaskExecutorManagerBuilder().setDefaultWorkerResourceSpec(workerResourceSpec).setNumSlotsPerWorker(// set to one so that the slot profiles directly correspond to
    1).setMaxNumSlots(2).createTaskExecutorManager()) {
        // create pending slot
        taskExecutorManager.allocateWorker(requestedSlotProfile);
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
        final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection();
        final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorConnection.getResourceID(), 0), requestedSlotProfile, JobID.generate(), new AllocationID()));
        taskExecutorManager.registerTaskManager(taskExecutorConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        // the slot from the task executor should be accepted, but we should still be waiting
        // for the originally requested slot
        assertThat(taskExecutorManager.getNumberRegisteredSlots(), is(1));
        assertThat(taskExecutorManager.getNumberPendingTaskManagerSlots(), is(1));
    }
}

Example 62 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class DefaultSlotStatusSyncerTest method testSlotStatusProcessing.

@Test
public void testSlotStatusProcessing() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final JobID jobId = new JobID();
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final SlotID slotId1 = new SlotID(taskExecutorConnection.getResourceID(), 0);
    final SlotID slotId2 = new SlotID(taskExecutorConnection.getResourceID(), 1);
    final SlotID slotId3 = new SlotID(taskExecutorConnection.getResourceID(), 2);
    final ResourceProfile totalResource = ResourceProfile.fromResources(5, 20);
    final ResourceProfile resource = ResourceProfile.fromResources(1, 4);
    final SlotReport slotReport1 = new SlotReport(Arrays.asList(new SlotStatus(slotId1, totalResource), new SlotStatus(slotId2, resource, jobId, allocationId1), new SlotStatus(slotId3, resource, jobId, allocationId2)));
    final SlotReport slotReport2 = new SlotReport(Arrays.asList(new SlotStatus(slotId3, resource), new SlotStatus(slotId2, resource, jobId, allocationId1)));
    taskManagerTracker.addTaskManager(taskExecutorConnection, totalResource, totalResource);
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport1);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", resource);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 3)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(2, 8)));
    final AllocationID allocationId3 = taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAllocatedSlots().keySet().stream().filter(allocationId -> !allocationId.equals(allocationId1) && !allocationId.equals(allocationId2)).findAny().get();
    // allocationId1 should still be allocated; allocationId2 should be freed; allocationId3
    // should continue to be in a pending state;
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport2);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertFalse(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).isPresent());
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).get().getState(), is(SlotState.ALLOCATED));
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).get().getState(), is(SlotState.PENDING));
}

Example 63 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class DefaultSlotStatusSyncerTest method testAllocateSlot.

@Test
public void testAllocateSlot() throws Exception {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> responseFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(tuple6);
        return responseFuture;
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    taskManagerTracker.addTaskManager(taskExecutorConnection, ResourceProfile.ANY, ResourceProfile.ANY);
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final JobID jobId = new JobID();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final CompletableFuture<Void> allocatedFuture = slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", ResourceProfile.ANY);
    final AllocationID allocationId = requestFuture.get().f2;
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(ResourceProfile.ANY, 1)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).isPresent());
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).get().getJobId(), is(jobId));
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId).get().getState(), is(SlotState.PENDING));
    responseFuture.complete(Acknowledge.get());
    assertFalse(allocatedFuture.isCompletedExceptionally());
}

Example 64 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FineGrainedTaskManagerRegistrationTest method testNotifyAllocationComplete.

@Test
public void testNotifyAllocationComplete() {
    final ResourceProfile totalResource = ResourceProfile.fromResources(10, 1000);
    final FineGrainedTaskManagerRegistration taskManager = new FineGrainedTaskManagerRegistration(TASK_EXECUTOR_CONNECTION, totalResource, totalResource);
    final AllocationID allocationId = new AllocationID();
    final JobID jobId = new JobID();
    final FineGrainedTaskManagerSlot slot = new FineGrainedTaskManagerSlot(allocationId, jobId, ResourceProfile.fromResources(2, 100), TASK_EXECUTOR_CONNECTION, SlotState.PENDING);
    taskManager.notifyAllocation(allocationId, slot);
    assertThat(taskManager.getAvailableResource(), is(ResourceProfile.fromResources(8, 900)));
    assertThat(taskManager.getIdleSince(), is(Long.MAX_VALUE));
    assertTrue(taskManager.getAllocatedSlots().containsKey(allocationId));
    taskManager.notifyAllocationComplete(allocationId);
    assertThat(taskManager.getAvailableResource(), is(ResourceProfile.fromResources(8, 900)));
    assertThat(taskManager.getIdleSince(), is(Long.MAX_VALUE));
    assertTrue(taskManager.getAllocatedSlots().containsKey(allocationId));
    assertThat(taskManager.getAllocatedSlots().get(allocationId).getState(), is(SlotState.ALLOCATED));
}

Example 65 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class TaskStateManagerImplTest method testForwardingSubtaskLocalStateBaseDirFromLocalStateStore.

/**
 * This tests if the {@link TaskStateManager} properly returns the subtask local state dir from
 * the corresponding {@link TaskLocalStateStoreImpl}.
 */
@Test
public void testForwardingSubtaskLocalStateBaseDirFromLocalStateStore() throws IOException {
    JobID jobID = new JobID(42L, 43L);
    AllocationID allocationID = new AllocationID(4711L, 23L);
    JobVertexID jobVertexID = new JobVertexID(12L, 34L);
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();
    Executor directExecutor = Executors.directExecutor();
    TemporaryFolder tmpFolder = new TemporaryFolder();
    try {
        tmpFolder.create();
        File[] allocBaseDirs = new File[] { tmpFolder.newFolder(), tmpFolder.newFolder(), tmpFolder.newFolder() };
        LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocBaseDirs, jobID, jobVertexID, 0);
        LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
        TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, 13, localRecoveryConfig, directExecutor);
        InMemoryStateChangelogStorage changelogStorage = new InMemoryStateChangelogStorage();
        TaskStateManager taskStateManager = taskStateManager(jobID, executionAttemptID, checkpointResponderMock, null, taskLocalStateStore, changelogStorage);
        LocalRecoveryConfig localRecoveryConfFromTaskLocalStateStore = taskLocalStateStore.getLocalRecoveryConfig();
        LocalRecoveryConfig localRecoveryConfFromTaskStateManager = taskStateManager.createLocalRecoveryConfig();
        for (int i = 0; i < 10; ++i) {
            Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskLocalStateStore.getLocalStateDirectoryProvider().get().allocationBaseDirectory(i));
            Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskStateManager.getLocalStateDirectoryProvider().get().allocationBaseDirectory(i));
        }
        Assert.assertEquals(localRecoveryConfFromTaskLocalStateStore.isLocalRecoveryEnabled(), localRecoveryConfFromTaskStateManager.isLocalRecoveryEnabled());
    } finally {
        tmpFolder.delete();
    }
}