Examples with SlotStatus org.apache.flink.runtime.taskexecutor.SlotStatus used on opensource projects

Example 16 with SlotStatus

use of org.apache.flink.runtime.taskexecutor.SlotStatus in project flink by apache.

the class DefaultSlotStatusSyncerTest method testSlotStatusProcessing.

@Test
public void testSlotStatusProcessing() {
    final FineGrainedTaskManagerTracker taskManagerTracker = new FineGrainedTaskManagerTracker();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final SlotStatusSyncer slotStatusSyncer = new DefaultSlotStatusSyncer(TASK_MANAGER_REQUEST_TIMEOUT);
    slotStatusSyncer.initialize(taskManagerTracker, resourceTracker, ResourceManagerId.generate(), TestingUtils.defaultExecutor());
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> new CompletableFuture<>()).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final JobID jobId = new JobID();
    final AllocationID allocationId1 = new AllocationID();
    final AllocationID allocationId2 = new AllocationID();
    final SlotID slotId1 = new SlotID(taskExecutorConnection.getResourceID(), 0);
    final SlotID slotId2 = new SlotID(taskExecutorConnection.getResourceID(), 1);
    final SlotID slotId3 = new SlotID(taskExecutorConnection.getResourceID(), 2);
    final ResourceProfile totalResource = ResourceProfile.fromResources(5, 20);
    final ResourceProfile resource = ResourceProfile.fromResources(1, 4);
    final SlotReport slotReport1 = new SlotReport(Arrays.asList(new SlotStatus(slotId1, totalResource), new SlotStatus(slotId2, resource, jobId, allocationId1), new SlotStatus(slotId3, resource, jobId, allocationId2)));
    final SlotReport slotReport2 = new SlotReport(Arrays.asList(new SlotStatus(slotId3, resource), new SlotStatus(slotId2, resource, jobId, allocationId1)));
    taskManagerTracker.addTaskManager(taskExecutorConnection, totalResource, totalResource);
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport1);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    slotStatusSyncer.allocateSlot(taskExecutorConnection.getInstanceID(), jobId, "address", resource);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 3)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(2, 8)));
    final AllocationID allocationId3 = taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAllocatedSlots().keySet().stream().filter(allocationId -> !allocationId.equals(allocationId1) && !allocationId.equals(allocationId2)).findAny().get();
    // allocationId1 should still be allocated; allocationId2 should be freed; allocationId3
    // should continue to be in a pending state;
    slotStatusSyncer.reportSlotStatus(taskExecutorConnection.getInstanceID(), slotReport2);
    assertThat(resourceTracker.getAcquiredResources(jobId), contains(ResourceRequirement.create(resource, 2)));
    assertThat(taskManagerTracker.getRegisteredTaskManager(taskExecutorConnection.getInstanceID()).get().getAvailableResource(), equalTo(ResourceProfile.fromResources(3, 12)));
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).isPresent());
    assertFalse(taskManagerTracker.getAllocatedOrPendingSlot(allocationId2).isPresent());
    assertTrue(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).isPresent());
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId1).get().getState(), is(SlotState.ALLOCATED));
    assertThat(taskManagerTracker.getAllocatedOrPendingSlot(allocationId3).get().getState(), is(SlotState.PENDING));
}

Example 17 with SlotStatus

use of org.apache.flink.runtime.taskexecutor.SlotStatus in project flink by apache.

the class SlotManager method registerTaskExecutor.

/**
	 * Registers a TaskExecutor
	 * @param resourceID TaskExecutor's ResourceID
	 * @param registration TaskExecutor's registration
	 * @param slotReport TaskExecutor's free and allocated slots
	 */
public void registerTaskExecutor(ResourceID resourceID, TaskExecutorRegistration registration, SlotReport slotReport) {
    if (taskManagers.get(resourceID) != null) {
        notifyTaskManagerFailure(resourceID);
    }
    this.taskManagers.put(resourceID, registration);
    for (SlotStatus slotStatus : slotReport.getSlotsStatus()) {
        final SlotID slotId = slotStatus.getSlotID();
        final TaskExecutorRegistration taskExecutorRegistration = taskManagers.get(slotId.getResourceID());
        if (taskExecutorRegistration == null) {
            LOG.info("Received SlotStatus but ResourceID {} is unknown to the SlotManager", slotId.getResourceID());
            return;
        }
        final ResourceSlot slot = new ResourceSlot(slotId, slotStatus.getProfiler(), taskExecutorRegistration);
        registerNewSlot(slot);
        LOG.info("New slot appeared, SlotID:{}, AllocationID:{}", slotId, slotStatus.getAllocationID());
        if (slotStatus.getAllocationID() != null) {
            // slot in use, record this in bookkeeping
            allocationMap.addAllocation(slotId, slotStatus.getAllocationID());
        } else {
            handleFreeSlot(slot);
        }
    }
}

Example 18 with SlotStatus

use of org.apache.flink.runtime.taskexecutor.SlotStatus in project flink by apache.

the class SlotManagerTest method testNewlyAppearedFreeSlotFulfillPendingRequest.

/**
	 * Tests that a new slot appeared in SlotReport, and we used it to fulfill a pending request
	 */
@Test
public void testNewlyAppearedFreeSlotFulfillPendingRequest() {
    TestingSlotManager slotManager = new TestingSlotManager();
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(1, slotManager.getPendingRequestCount());
    SlotID slotId = SlotID.generate();
    SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
    SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    assertTrue(slotManager.isAllocated(slotId));
}

Example 19 with SlotStatus

use of org.apache.flink.runtime.taskexecutor.SlotStatus in project flink by apache.

the class SlotManagerTest method testNewlyAppearedInUseSlot.

/**
	 * Tests that a new slot appeared in SlotReport, and it's been reported using by some job
	 */
@Test
public void testNewlyAppearedInUseSlot() {
    TestingSlotManager slotManager = new TestingSlotManager();
    SlotID slotId = SlotID.generate();
    SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
    SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertTrue(slotManager.isAllocated(slotId));
}

Example 20 with SlotStatus

use of org.apache.flink.runtime.taskexecutor.SlotStatus in project flink by apache.

the class SlotProtocolTest method testSlotsUnavailableRequest.

/**
	 * Tests whether
	 * 1) SlotRequest is routed to the SlotManager
	 * 2) SlotRequest is confirmed
	 * 3) SlotRequest leads to a container allocation
	 * 4) Slot becomes available and TaskExecutor gets a SlotRequest
	 */
@Test
public void testSlotsUnavailableRequest() throws Exception {
    final String rmAddress = "/rm1";
    final String jmAddress = "/jm1";
    final JobID jobID = new JobID();
    testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
    final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
    final UUID rmLeaderID = UUID.randomUUID();
    final UUID jmLeaderID = UUID.randomUUID();
    TestingLeaderElectionService rmLeaderElectionService = configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices, testRpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
    SpiedResourceManager resourceManager = new SpiedResourceManager(testRpcService, resourceManagerConfiguration, testingHaServices, slotManagerFactory, mock(MetricRegistry.class), jobLeaderIdService, mock(FatalErrorHandler.class));
    resourceManager.start();
    rmLeaderElectionService.isLeader(rmLeaderID);
    Future<RegistrationResponse> registrationFuture = resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
    try {
        registrationFuture.get(5, TimeUnit.SECONDS);
    } catch (Exception e) {
        Assert.fail("JobManager registration Future didn't become ready.");
    }
    final SlotManager slotManager = slotManagerFactory.slotManager;
    final AllocationID allocationID = new AllocationID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
    SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
    RMSlotRequestReply slotRequestReply = resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
    // 1) SlotRequest is routed to the SlotManager
    verify(slotManager).requestSlot(slotRequest);
    // 2) SlotRequest is confirmed
    Assert.assertEquals(slotRequestReply.getAllocationID(), allocationID);
    // 3) SlotRequest leads to a container allocation
    Assert.assertEquals(1, resourceManager.startNewWorkerCalled);
    Assert.assertFalse(slotManager.isAllocated(allocationID));
    // slot becomes available
    final String tmAddress = "/tm1";
    TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
    Mockito.when(taskExecutorGateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
    testRpcService.registerGateway(tmAddress, taskExecutorGateway);
    final ResourceID resourceID = ResourceID.generate();
    final SlotID slotID = new SlotID(resourceID, 0);
    final SlotStatus slotStatus = new SlotStatus(slotID, resourceProfile);
    final SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    // register slot at SlotManager
    slotManager.registerTaskExecutor(resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
    // 4) Slot becomes available and TaskExecutor gets a SlotRequest
    verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}