Examples with AllocationID org.apache.flink.runtime.clusterframework.types.AllocationID used on opensource projects

Example 76 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class TaskExecutorTest method testJobLeaderDetection.

/**
 * Tests that a TaskManager detects a job leader for which it has reserved slots. Upon detecting
 * the job leader, it will offer all reserved slots to the JobManager.
 */
@Test
public void testJobLeaderDetection() throws Exception {
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);
    final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
    final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
    CompletableFuture<Void> initialSlotReportFuture = new CompletableFuture<>();
    resourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        initialSlotReportFuture.complete(null);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final CompletableFuture<Collection<SlotOffer>> offeredSlotsFuture = new CompletableFuture<>();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        offeredSlotsFuture.complete(new ArrayList<>(slotOffers));
        return CompletableFuture.completedFuture(slotOffers);
    }).build();
    rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
    rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
    final AllocationID allocationId = new AllocationID();
    final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation).setTaskSlotTable(taskSlotTable).setJobLeaderService(jobLeaderService).setTaskStateManager(localStateStoresManager).build();
    TaskExecutor taskManager = createTaskExecutor(taskManagerServices);
    try {
        taskManager.start();
        final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);
        // tell the task manager about the rm leader
        resourceManagerLeaderRetriever.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());
        initialSlotReportFuture.get();
        requestSlot(tmGateway, jobId, allocationId, buildSlotID(0), ResourceProfile.ZERO, jobMasterGateway.getAddress(), resourceManagerGateway.getFencingToken());
        // now inform the task manager about the new job leader
        jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
        final Collection<SlotOffer> offeredSlots = offeredSlotsFuture.get();
        final Collection<AllocationID> allocationIds = offeredSlots.stream().map(SlotOffer::getAllocationId).collect(Collectors.toList());
        assertThat(allocationIds, containsInAnyOrder(allocationId));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskManager, timeout);
    }
}

Example 77 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class TaskExecutorTest method testOfferSlotToJobMasterAfterTimeout.

/**
 * Tests that offers slots to job master timeout and retry.
 */
@Test
public void testOfferSlotToJobMasterAfterTimeout() throws Exception {
    final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();
    final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
    final AllocationID allocationId = new AllocationID();
    final CompletableFuture<ResourceID> initialSlotReportFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
    testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        initialSlotReportFuture.complete(null);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
    resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());
    final CountDownLatch slotOfferings = new CountDownLatch(3);
    final CompletableFuture<AllocationID> offeredSlotFuture = new CompletableFuture<>();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setOfferSlotsFunction((resourceID, slotOffers) -> {
        assertThat(slotOffers.size(), is(1));
        slotOfferings.countDown();
        if (slotOfferings.getCount() == 0) {
            offeredSlotFuture.complete(slotOffers.iterator().next().getAllocationId());
            return CompletableFuture.completedFuture(slotOffers);
        } else {
            return FutureUtils.completedExceptionally(new TimeoutException());
        }
    }).build();
    final String jobManagerAddress = jobMasterGateway.getAddress();
    rpc.registerGateway(jobManagerAddress, jobMasterGateway);
    jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobMasterGateway.getFencingToken().toUUID());
    try {
        taskExecutor.start();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        // wait for the connection to the ResourceManager
        initialSlotReportFuture.get();
        requestSlot(taskExecutorGateway, jobId, allocationId, new SlotID(taskExecutor.getResourceID(), 0), ResourceProfile.ZERO, jobManagerAddress, testingResourceManagerGateway.getFencingToken());
        slotOfferings.await();
        assertThat(offeredSlotFuture.get(), is(allocationId));
        assertTrue(taskSlotTable.isSlotFree(1));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}

Example 78 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class DefaultTimerServiceTest method testUnregisterAllTimeouts.

/**
 * Tests that all registered timeouts can be unregistered.
 */
@Test
public void testUnregisterAllTimeouts() throws Exception {
    final ManuallyTriggeredScheduledExecutorService scheduledExecutorService = new ManuallyTriggeredScheduledExecutorService();
    DefaultTimerService<AllocationID> timerService = new DefaultTimerService<>(scheduledExecutorService, 100L);
    timerService.start((ignoredA, ignoredB) -> {
    });
    timerService.registerTimeout(new AllocationID(), 10, TimeUnit.SECONDS);
    timerService.registerTimeout(new AllocationID(), 10, TimeUnit.SECONDS);
    timerService.unregisterAllTimeouts();
    Map<?, ?> timeouts = timerService.getTimeouts();
    assertTrue(timeouts.isEmpty());
    for (ScheduledFuture<?> scheduledTask : scheduledExecutorService.getAllScheduledTasks()) {
        assertThat(scheduledTask.isCancelled(), is(true));
    }
}

Example 79 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class DefaultTimerServiceTest method testIsValidReturnsTrueForActiveTimeout.

@Test
public void testIsValidReturnsTrueForActiveTimeout() throws Exception {
    final DefaultTimerService<AllocationID> timerService = createAndStartTimerService();
    final AllocationID allocationId = new AllocationID();
    timerService.registerTimeout(allocationId, 10, TimeUnit.SECONDS);
    final DefaultTimerService.Timeout<AllocationID> timeout = timerService.getTimeouts().get(allocationId);
    final UUID ticket = timeout.getTicket();
    assertThat(timerService.isValid(allocationId, ticket), is(true));
}

Example 80 with AllocationID

use of org.apache.flink.runtime.clusterframework.types.AllocationID in project flink by apache.

the class FileSlotAllocationSnapshotPersistenceServiceTest method createRandomSlotAllocationSnapshots.

@Nonnull
private Collection<SlotAllocationSnapshot> createRandomSlotAllocationSnapshots(int number) {
    final Collection<SlotAllocationSnapshot> result = new ArrayList<>();
    final ResourceID resourceId = ResourceID.generate();
    for (int slotIndex = 0; slotIndex < number; slotIndex++) {
        result.add(new SlotAllocationSnapshot(new SlotID(resourceId, slotIndex), new JobID(), "foobar", new AllocationID(), ResourceProfile.UNKNOWN));
    }
    return result;
}