Examples with SlotRequest org.apache.flink.runtime.resourcemanager.SlotRequest used on opensource projects

Example 16 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotPoolTest method testReleaseResource.

@Test
public void testReleaseResource() throws Exception {
    ResourceID resourceID = new ResourceID("resource");
    slotPool.registerTaskManager(resourceID);
    Future<SimpleSlot> future1 = slotPool.allocateSlot(mock(ScheduledUnit.class), DEFAULT_TESTING_PROFILE, null);
    ArgumentCaptor<SlotRequest> slotRequestArgumentCaptor = ArgumentCaptor.forClass(SlotRequest.class);
    verify(resourceManagerGateway).requestSlot(any(UUID.class), any(UUID.class), slotRequestArgumentCaptor.capture(), any(Time.class));
    final SlotRequest slotRequest = slotRequestArgumentCaptor.getValue();
    Future<SimpleSlot> future2 = slotPool.allocateSlot(mock(ScheduledUnit.class), DEFAULT_TESTING_PROFILE, null);
    AllocatedSlot allocatedSlot = createAllocatedSlot(resourceID, slotRequest.getAllocationId(), jobId, DEFAULT_TESTING_PROFILE);
    assertTrue(slotPool.offerSlot(allocatedSlot));
    SimpleSlot slot1 = future1.get(1, TimeUnit.SECONDS);
    assertTrue(future1.isDone());
    assertFalse(future2.isDone());
    slotPool.releaseTaskManager(resourceID);
    assertTrue(slot1.isReleased());
    // slot released and not usable, second allocation still not fulfilled
    Thread.sleep(10);
    assertFalse(future2.isDone());
}

Example 17 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testNewlyAppearedFreeSlotFulfillPendingRequest.

/**
	 * Tests that a new slot appeared in SlotReport, and we used it to fulfill a pending request
	 */
@Test
public void testNewlyAppearedFreeSlotFulfillPendingRequest() {
    TestingSlotManager slotManager = new TestingSlotManager();
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(1, slotManager.getPendingRequestCount());
    SlotID slotId = SlotID.generate();
    SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
    SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
    slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    assertTrue(slotManager.isAllocated(slotId));
}

Example 18 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testRequestSlotWithoutSuitableSlot.

/**
	 * Tests that there are some free slots when we request, but none of them are suitable
	 */
@Test
public void testRequestSlotWithoutSuitableSlot() {
    TestingSlotManager slotManager = new TestingSlotManager();
    directlyProvideFreeSlots(slotManager, DEFAULT_TESTING_PROFILE, 2);
    assertEquals(2, slotManager.getFreeSlotCount());
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_BIG_PROFILE));
    assertEquals(0, slotManager.getAllocatedSlotCount());
    assertEquals(2, slotManager.getFreeSlotCount());
    assertEquals(1, slotManager.getPendingRequestCount());
    assertEquals(1, slotManager.getAllocatedContainers().size());
    assertEquals(DEFAULT_TESTING_BIG_PROFILE, slotManager.getAllocatedContainers().get(0));
}

Example 19 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class SlotManagerTest method testNotifyTaskManagerFailure.

@Test
public void testNotifyTaskManagerFailure() {
    TestingSlotManager slotManager = new TestingSlotManager();
    ResourceID resource1 = ResourceID.generate();
    ResourceID resource2 = ResourceID.generate();
    ResourceSlot slot11 = new ResourceSlot(new SlotID(resource1, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot12 = new ResourceSlot(new SlotID(resource1, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot21 = new ResourceSlot(new SlotID(resource2, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    ResourceSlot slot22 = new ResourceSlot(new SlotID(resource2, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
    slotManager.addFreeSlot(slot11);
    slotManager.addFreeSlot(slot21);
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    slotManager.requestSlot(new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE));
    assertEquals(2, slotManager.getAllocatedSlotCount());
    assertEquals(0, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    slotManager.addFreeSlot(slot12);
    slotManager.addFreeSlot(slot22);
    assertEquals(2, slotManager.getAllocatedSlotCount());
    assertEquals(2, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    slotManager.notifyTaskManagerFailure(resource2);
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
    // notify an not exist resource failure
    slotManager.notifyTaskManagerFailure(ResourceID.generate());
    assertEquals(1, slotManager.getAllocatedSlotCount());
    assertEquals(1, slotManager.getFreeSlotCount());
    assertEquals(0, slotManager.getPendingRequestCount());
}

Example 20 with SlotRequest

use of org.apache.flink.runtime.resourcemanager.SlotRequest in project flink by apache.

the class TaskExecutorITCase method testSlotAllocation.

@Test
public void testSlotAllocation() throws Exception {
    TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
    TestingHighAvailabilityServices testingHAServices = new TestingHighAvailabilityServices();
    final Configuration configuration = new Configuration();
    final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
    final ResourceID taskManagerResourceId = new ResourceID("foobar");
    final UUID rmLeaderId = UUID.randomUUID();
    final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
    final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService();
    final String rmAddress = "rm";
    final String jmAddress = "jm";
    final UUID jmLeaderId = UUID.randomUUID();
    final JobID jobId = new JobID();
    final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
    testingHAServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
    testingHAServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
    testingHAServices.setJobMasterLeaderRetriever(jobId, new TestingLeaderRetrievalService(jmAddress, jmLeaderId));
    TestingSerialRpcService rpcService = new TestingSerialRpcService();
    ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.milliseconds(500L), Time.milliseconds(500L), Time.minutes(5L));
    SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHAServices, rpcService.getScheduledExecutor(), resourceManagerConfiguration.getJobTimeout());
    MetricRegistry metricRegistry = mock(MetricRegistry.class);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class, RETURNS_MOCKS);
    final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(taskManagerResourceId, InetAddress.getLocalHost(), 1234);
    final MemoryManager memoryManager = mock(MemoryManager.class);
    final IOManager ioManager = mock(IOManager.class);
    final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
    final BroadcastVariableManager broadcastVariableManager = mock(BroadcastVariableManager.class);
    final FileCache fileCache = mock(FileCache.class);
    final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(resourceProfile), new TimerService<AllocationID>(scheduledExecutorService, 100L));
    final JobManagerTable jobManagerTable = new JobManagerTable();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    ResourceManager<ResourceID> resourceManager = new StandaloneResourceManager(rpcService, resourceManagerConfiguration, testingHAServices, slotManagerFactory, metricRegistry, jobLeaderIdService, testingFatalErrorHandler);
    TaskExecutor taskExecutor = new TaskExecutor(taskManagerConfiguration, taskManagerLocation, rpcService, memoryManager, ioManager, networkEnvironment, testingHAServices, heartbeatServices, metricRegistry, taskManagerMetricGroup, broadcastVariableManager, fileCache, taskSlotTable, jobManagerTable, jobLeaderService, testingFatalErrorHandler);
    JobMasterGateway jmGateway = mock(JobMasterGateway.class);
    when(jmGateway.registerTaskManager(any(String.class), any(TaskManagerLocation.class), eq(jmLeaderId), any(Time.class))).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(taskManagerResourceId, 1234)));
    when(jmGateway.getHostname()).thenReturn(jmAddress);
    rpcService.registerGateway(rmAddress, resourceManager.getSelf());
    rpcService.registerGateway(jmAddress, jmGateway);
    final AllocationID allocationId = new AllocationID();
    final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile);
    final SlotOffer slotOffer = new SlotOffer(allocationId, 0, resourceProfile);
    try {
        resourceManager.start();
        taskExecutor.start();
        // notify the RM that it is the leader
        rmLeaderElectionService.isLeader(rmLeaderId);
        // notify the TM about the new RM leader
        rmLeaderRetrievalService.notifyListener(rmAddress, rmLeaderId);
        Future<RegistrationResponse> registrationResponseFuture = resourceManager.registerJobManager(rmLeaderId, jmLeaderId, jmAddress, jobId);
        RegistrationResponse registrationResponse = registrationResponseFuture.get();
        assertTrue(registrationResponse instanceof JobMasterRegistrationSuccess);
        resourceManager.requestSlot(jmLeaderId, rmLeaderId, slotRequest);
        verify(jmGateway).offerSlots(eq(taskManagerResourceId), (Iterable<SlotOffer>) argThat(Matchers.contains(slotOffer)), eq(jmLeaderId), any(Time.class));
    } finally {
        if (testingFatalErrorHandler.hasExceptionOccurred()) {
            testingFatalErrorHandler.rethrowError();
        }
    }
}