Examples with AllocationID org.apache.flink.runtime.clusterframework.types.AllocationID used on opensource projects

Example 31 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class AllocatedSlotsTest method testOperations.

@Test
public void testOperations() throws Exception {
    SlotPool.AllocatedSlots allocatedSlots = new SlotPool.AllocatedSlots();
    final AllocationID allocation1 = new AllocationID();
    final ResourceID resource1 = new ResourceID("resource1");
    final Slot slot1 = createSlot(resource1, allocation1);
    allocatedSlots.add(slot1);
    assertTrue(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.containResource(resource1));
    assertEquals(slot1, allocatedSlots.get(allocation1));
    assertEquals(1, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(1, allocatedSlots.size());
    final AllocationID allocation2 = new AllocationID();
    final Slot slot2 = createSlot(resource1, allocation2);
    allocatedSlots.add(slot2);
    assertTrue(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.contains(slot2.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.containResource(resource1));
    assertEquals(slot1, allocatedSlots.get(allocation1));
    assertEquals(slot2, allocatedSlots.get(allocation2));
    assertEquals(2, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(2, allocatedSlots.size());
    final AllocationID allocation3 = new AllocationID();
    final ResourceID resource2 = new ResourceID("resource2");
    final Slot slot3 = createSlot(resource2, allocation3);
    allocatedSlots.add(slot3);
    assertTrue(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.contains(slot2.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.contains(slot3.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.containResource(resource1));
    assertTrue(allocatedSlots.containResource(resource2));
    assertEquals(slot1, allocatedSlots.get(allocation1));
    assertEquals(slot2, allocatedSlots.get(allocation2));
    assertEquals(slot3, allocatedSlots.get(allocation3));
    assertEquals(2, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(1, allocatedSlots.getSlotsForTaskManager(resource2).size());
    assertEquals(3, allocatedSlots.size());
    allocatedSlots.remove(slot2);
    assertTrue(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.contains(slot2.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.contains(slot3.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.containResource(resource1));
    assertTrue(allocatedSlots.containResource(resource2));
    assertEquals(slot1, allocatedSlots.get(allocation1));
    assertNull(allocatedSlots.get(allocation2));
    assertEquals(slot3, allocatedSlots.get(allocation3));
    assertEquals(1, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(1, allocatedSlots.getSlotsForTaskManager(resource2).size());
    assertEquals(2, allocatedSlots.size());
    allocatedSlots.remove(slot1);
    assertFalse(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.contains(slot2.getAllocatedSlot().getSlotAllocationId()));
    assertTrue(allocatedSlots.contains(slot3.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.containResource(resource1));
    assertTrue(allocatedSlots.containResource(resource2));
    assertNull(allocatedSlots.get(allocation1));
    assertNull(allocatedSlots.get(allocation2));
    assertEquals(slot3, allocatedSlots.get(allocation3));
    assertEquals(0, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(1, allocatedSlots.getSlotsForTaskManager(resource2).size());
    assertEquals(1, allocatedSlots.size());
    allocatedSlots.remove(slot3);
    assertFalse(allocatedSlots.contains(slot1.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.contains(slot2.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.contains(slot3.getAllocatedSlot().getSlotAllocationId()));
    assertFalse(allocatedSlots.containResource(resource1));
    assertFalse(allocatedSlots.containResource(resource2));
    assertNull(allocatedSlots.get(allocation1));
    assertNull(allocatedSlots.get(allocation2));
    assertNull(allocatedSlots.get(allocation3));
    assertEquals(0, allocatedSlots.getSlotsForTaskManager(resource1).size());
    assertEquals(0, allocatedSlots.getSlotsForTaskManager(resource2).size());
    assertEquals(0, allocatedSlots.size());
}

Example 32 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class SlotProtocolTest method testSlotsUnavailableRequest.

/**
	 * Tests whether
	 * 1) SlotRequest is routed to the SlotManager
	 * 2) SlotRequest is confirmed
	 * 3) SlotRequest leads to a container allocation
	 * 4) Slot becomes available and TaskExecutor gets a SlotRequest
	 */
@Test
public void testSlotsUnavailableRequest() throws Exception {
    final String rmAddress = "/rm1";
    final String jmAddress = "/jm1";
    final JobID jobID = new JobID();
    testRpcService.registerGateway(jmAddress, mock(JobMasterGateway.class));
    final TestingHighAvailabilityServices testingHaServices = new TestingHighAvailabilityServices();
    final UUID rmLeaderID = UUID.randomUUID();
    final UUID jmLeaderID = UUID.randomUUID();
    TestingLeaderElectionService rmLeaderElectionService = configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L), Time.minutes(5L));
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices, testRpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
    SpiedResourceManager resourceManager = new SpiedResourceManager(testRpcService, resourceManagerConfiguration, testingHaServices, slotManagerFactory, mock(MetricRegistry.class), jobLeaderIdService, mock(FatalErrorHandler.class));
    resourceManager.start();
    rmLeaderElectionService.isLeader(rmLeaderID);
    Future<RegistrationResponse> registrationFuture = resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
    try {
        registrationFuture.get(5, TimeUnit.SECONDS);
    } catch (Exception e) {
        Assert.fail("JobManager registration Future didn't become ready.");
    }
    final SlotManager slotManager = slotManagerFactory.slotManager;
    final AllocationID allocationID = new AllocationID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
    SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
    RMSlotRequestReply slotRequestReply = resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
    // 1) SlotRequest is routed to the SlotManager
    verify(slotManager).requestSlot(slotRequest);
    // 2) SlotRequest is confirmed
    Assert.assertEquals(slotRequestReply.getAllocationID(), allocationID);
    // 3) SlotRequest leads to a container allocation
    Assert.assertEquals(1, resourceManager.startNewWorkerCalled);
    Assert.assertFalse(slotManager.isAllocated(allocationID));
    // slot becomes available
    final String tmAddress = "/tm1";
    TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
    Mockito.when(taskExecutorGateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class))).thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
    testRpcService.registerGateway(tmAddress, taskExecutorGateway);
    final ResourceID resourceID = ResourceID.generate();
    final SlotID slotID = new SlotID(resourceID, 0);
    final SlotStatus slotStatus = new SlotStatus(slotID, resourceProfile);
    final SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    // register slot at SlotManager
    slotManager.registerTaskExecutor(resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
    // 4) Slot becomes available and TaskExecutor gets a SlotRequest
    verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}

Example 33 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class SlotManagerTest method testNewlyAppearedFreeSlotFulfillPendingRequest.

/**
	 * Tests that a new slot appeared in SlotReport, and we used it to fulfill a pending request
	 */
@Test
public void testNewlyAppearedFreeSlotFulfillPendingRequest() {
    TestingSlotManager slotManager = new TestingSlotManager();
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(1, slotManager.getPendingRequestCount());
    SlotID slotId = SlotID.generate();
    SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
    SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    assertTrue(slotManager.isAllocated(slotId));
}

Example 34 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class SlotManagerTest method testRequestSlotWithoutSuitableSlot.

/**
	 * Tests that there are some free slots when we request, but none of them are suitable
	 */
@Test
public void testRequestSlotWithoutSuitableSlot() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 2);
    assertEquals(2, slotManager.getFreeSlotCount());
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
    assertEquals(0, slotManager.getAllocatedSlotCount());
    assertEquals(2, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 35 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class SlotManagerTest method testNotifyTaskManagerFailure.

@Test
public void testNotifyTaskManagerFailure() {
    TestingSlotManager slotManager = new TestingSlotManager();
    ResourceID resource1 = ResourceID.generate();
    ResourceID resource2 = ResourceID.generate();
    ResourceSlot slot11 = new ResourceSlot(new SlotID(resource1, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot12 = new ResourceSlot(new SlotID(resource1, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot21 = new ResourceSlot(new SlotID(resource2, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot22 = new ResourceSlot(new SlotID(resource2, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    slotManager.addFreeSlot(slot11);
    slotManager.addFreeSlot(slot21);
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(2, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    slotManager.addFreeSlot(slot12);
    slotManager.addFreeSlot(slot22);
    assertEquals(2, slotManager.getAllocatedSlotCount());
    assertEquals(2, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    slotManager.notifyTaskManagerFailure(resource2);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    // notify an not exist resource failure
    slotManager.notifyTaskManagerFailure(ResourceID.generate());
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
}