Examples with SlotRequest org.apache.flink.runtime.resourcemanager.SlotRequest used on opensource projects

Example 6 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotProtocolTest method testSlotAvailableRequest.

/**
	 * Tests whether
	 * 1) a SlotRequest is routed to the SlotManager
	 * 2) a SlotRequest is confirmed
	 * 3) a SlotRequest leads to an allocation of a registered slot
	 * 4) a SlotRequest is routed to the TaskExecutor
	 */
@Test
public void testSlotAvailableRequest() throws Exception {
    final String rmAddress = "/rm1";
    final String jmAddress = "/jm1";
    final String tmAddress = "/tm1";
    final JobID jobID = new JobID();
    testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
    final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
    final UUID rmLeaderID = UUID.randomUUID();
    final UUID jmLeaderID = UUID.randomUUID();
    TestingLeaderElectionService rmLeaderElectionService = configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
    TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
    Mockito.when(taskExecutorGateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
    testRpcService.registerGateway(tmAddress, taskExecutorGateway);
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices, testRpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
    ResourceManager<ResourceID> resourceManager = Mockito.spy(new StandaloneResourceManager(testRpcService, resourceManagerConfiguration, testingHaServices, slotManagerFactory, mock(MetricRegistry.class), jobLeaderIdService, mock(FatalErrorHandler.class)));
    resourceManager.start();
    rmLeaderElectionService.isLeader(rmLeaderID);
    Thread.sleep(1000);
    Future<RegistrationResponse> registrationFuture = resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
    try {
        registrationFuture.get(5L, TimeUnit.SECONDS);
    } catch (Exception e) {
        Assert.fail("JobManager registration Future didn't become ready.");
    }
    final SlotManager slotManager = slotManagerFactory.slotManager;
    final ResourceID resourceID = ResourceID.generate();
    final AllocationID allocationID = new AllocationID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
    final SlotID slotID = new SlotID(resourceID, 0);
    final SlotStatus slotStatus = new SlotStatus(slotID, resourceProfile);
    final SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    // register slot at SlotManager
    slotManager.registerTaskExecutor(resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
    SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
    RMSlotRequestReply slotRequestReply = resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
    // 1) a SlotRequest is routed to the SlotManager
    verify(slotManager).requestSlot(slotRequest);
    // 2) a SlotRequest is confirmed
    Assert.assertEquals(slotRequestReply.getAllocationID(), allocationID);
    // 3) a SlotRequest leads to an allocation of a registered slot
    Assert.assertTrue(slotManager.isAllocated(slotID));
    Assert.assertTrue(slotManager.isAllocated(allocationID));
    // 4) a SlotRequest is routed to the TaskExecutor
    verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}

Example 7 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testRequestSlotWithoutFreeSlot.

/**
	 * Tests that there are no free slots when we request, need to allocate from cluster manager master
	 */
@Test
public void testRequestSlotWithoutFreeSlot() {
    TestingSlotManager slotManager = new TestingSlotManager();
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(0, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 8 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testDuplicatedSlotRequest.

/**
	 * Tests that we send duplicated slot request
	 */
@Test
public void testDuplicatedSlotRequest() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 1);
    SlotRequest request1 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
    SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE);
    slotManager.requestSlot(request1);
    slotManager.requestSlot(request2);
    slotManager.requestSlot(request2);
    slotManager.requestSlot(request1);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 9 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testRequestMultipleSlots.

/**
	 * Tests that we send multiple slot requests
	 */
@Test
public void testRequestMultipleSlots() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 5);
    // request 3 normal slots
    for (int i = 0; i < 3; ++i) {
        slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    }
    // request 2 big slots
    for (int i = 0; i < 2; ++i) {
        slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
    }
    // request 1 normal slot again
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(4, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(2, slotManager.getPendingRequestCount());
    assertEquals(2, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(1));
}

Example 10 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testSlotAllocationFailedAtTaskManagerOccupiedByOther.

/**
	 * Tests that we did some allocation but failed / rejected by TaskManager, and slot is occupied by another request
	 * This can only occur after reconnect of the TaskExecutor.
	 */
@Test
public void testSlotAllocationFailedAtTaskManagerOccupiedByOther() {
    TestingSlotManager slotManager = new TestingSlotManager();
    final SlotID slotID = SlotID.generate();
    SlotStatus slot = new SlotStatus(slotID, DEFAULT_TESTING_PROFILE);
    SlotReport slotReport = new SlotReport(slot);
    slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
    SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
    slotManager.requestSlot(request);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    // slot is set empty by a reconnect of the TaskExecutor
    slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
    assertEquals(0, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    // another request takes the slot
    SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
    slotManager.requestSlot(request2);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    assertFalse(slotManager.isAllocated(request.getAllocationId()));
    assertTrue(slotManager.isAllocated(request2.getAllocationId()));
    // original request should be retried
    slotManager.handleSlotRequestFailedAtTaskManager(request, slotID);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    assertFalse(slotManager.isAllocated(request.getAllocationId()));
    assertTrue(slotManager.isAllocated(request2.getAllocationId()));
}