Examples with AllocatedSlotReport org.apache.flink.runtime.jobmaster.AllocatedSlotReport used on opensource projects

Example 1 with AllocatedSlotReport

use of org.apache.flink.runtime.jobmaster.AllocatedSlotReport in project flink by apache.

the class DeclarativeSlotPoolService method createAllocatedSlotReport.

@Override
public AllocatedSlotReport createAllocatedSlotReport(ResourceID taskManagerId) {
    assertHasBeenStarted();
    final Collection<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>();
    for (SlotInfo slotInfo : declarativeSlotPool.getAllSlotsInformation()) {
        if (slotInfo.getTaskManagerLocation().getResourceID().equals(taskManagerId)) {
            allocatedSlotInfos.add(new AllocatedSlotInfo(slotInfo.getPhysicalSlotNumber(), slotInfo.getAllocationId()));
        }
    }
    return new AllocatedSlotReport(jobId, allocatedSlotInfos);
}

Example 2 with AllocatedSlotReport

use of org.apache.flink.runtime.jobmaster.AllocatedSlotReport in project flink by apache.

the class DeclarativeSlotPoolServiceTest method testCreateAllocatedSlotReport.

@Test
public void testCreateAllocatedSlotReport() throws Exception {
    final LocalTaskManagerLocation taskManagerLocation1 = new LocalTaskManagerLocation();
    final LocalTaskManagerLocation taskManagerLocation2 = new LocalTaskManagerLocation();
    final SimpleSlotContext simpleSlotContext2 = createSimpleSlotContext(taskManagerLocation2);
    final Collection<SlotInfo> slotInfos = Arrays.asList(createSimpleSlotContext(taskManagerLocation1), simpleSlotContext2);
    try (DeclarativeSlotPoolService declarativeSlotPoolService = createDeclarativeSlotPoolService(new TestingDeclarativeSlotPoolFactory(new TestingDeclarativeSlotPoolBuilder().setGetAllSlotsInformationSupplier(() -> slotInfos)))) {
        final AllocatedSlotReport allocatedSlotReport = declarativeSlotPoolService.createAllocatedSlotReport(taskManagerLocation2.getResourceID());
        assertThat(allocatedSlotReport.getAllocatedSlotInfos(), contains(matchesWithSlotContext(simpleSlotContext2)));
    }
}

Example 3 with AllocatedSlotReport

use of org.apache.flink.runtime.jobmaster.AllocatedSlotReport in project flink by apache.

the class TaskExecutorTest method testJobManagerBecomesUnreachableTriggersDisconnect.

@Test
public void testJobManagerBecomesUnreachableTriggersDisconnect() throws Exception {
    final ResourceID jmResourceId = ResourceID.generate();
    runJobManagerHeartbeatTest(jmResourceId, failedRpcEnabledHeartbeatServices, jobMasterGatewayBuilder -> jobMasterGatewayBuilder.setTaskManagerHeartbeatFunction((resourceID, taskExecutorToJobManagerHeartbeatPayload) -> FutureUtils.completedExceptionally(new RecipientUnreachableException("sender", "recipient", "job manager is unreachable."))), (ignoredTaskExecutorResourceId, taskExecutorGateway, allocationId) -> taskExecutorGateway.heartbeatFromJobManager(jmResourceId, new AllocatedSlotReport(jobId, Collections.singleton(new AllocatedSlotInfo(0, allocationId)))));
}

Example 4 with AllocatedSlotReport

use of org.apache.flink.runtime.jobmaster.AllocatedSlotReport in project flink by apache.

the class TaskExecutorTest method testSyncSlotsWithJobMasterByHeartbeat.

/**
 * Tests that the TaskExecutor syncs its slots view with the JobMaster's view via the
 * AllocatedSlotReport reported by the heartbeat (See FLINK-11059).
 */
@Test
public void testSyncSlotsWithJobMasterByHeartbeat() throws Exception {
    final CountDownLatch activeSlots = new CountDownLatch(2);
    final TaskSlotTable<Task> taskSlotTable = new ActivateSlotNotifyingTaskSlotTable(2, activeSlots);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();
    final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
    final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
    final BlockingQueue<AllocationID> allocationsNotifiedFree = new ArrayBlockingQueue<>(2);
    OneShotLatch initialSlotReporting = new OneShotLatch();
    testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
        initialSlotReporting.trigger();
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    testingResourceManagerGateway.setNotifySlotAvailableConsumer(instanceIDSlotIDAllocationIDTuple3 -> allocationsNotifiedFree.offer(instanceIDSlotIDAllocationIDTuple3.f2));
    rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
    resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());
    final BlockingQueue<AllocationID> failedSlotFutures = new ArrayBlockingQueue<>(2);
    final ResourceID jobManagerResourceId = ResourceID.generate();
    final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setFailSlotConsumer((resourceID, allocationID, throwable) -> failedSlotFutures.offer(allocationID)).setOfferSlotsFunction((resourceID, slotOffers) -> CompletableFuture.completedFuture(new ArrayList<>(slotOffers))).setRegisterTaskManagerFunction((ignoredJobId, ignoredTaskManagerRegistrationInformation) -> CompletableFuture.completedFuture(new JMTMRegistrationSuccess(jobManagerResourceId))).build();
    final String jobManagerAddress = jobMasterGateway.getAddress();
    rpc.registerGateway(jobManagerAddress, jobMasterGateway);
    jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobMasterGateway.getFencingToken().toUUID());
    taskExecutor.start();
    try {
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        initialSlotReporting.await();
        final AllocationID allocationIdInBoth = new AllocationID();
        final AllocationID allocationIdOnlyInJM = new AllocationID();
        final AllocationID allocationIdOnlyInTM = new AllocationID();
        taskExecutorGateway.requestSlot(new SlotID(taskExecutor.getResourceID(), 0), jobId, allocationIdInBoth, ResourceProfile.ZERO, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);
        taskExecutorGateway.requestSlot(new SlotID(taskExecutor.getResourceID(), 1), jobId, allocationIdOnlyInTM, ResourceProfile.ZERO, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);
        activeSlots.await();
        List<AllocatedSlotInfo> allocatedSlotInfos = Arrays.asList(new AllocatedSlotInfo(0, allocationIdInBoth), new AllocatedSlotInfo(1, allocationIdOnlyInJM));
        AllocatedSlotReport allocatedSlotReport = new AllocatedSlotReport(jobId, allocatedSlotInfos);
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, allocatedSlotReport);
        assertThat(failedSlotFutures.take(), is(allocationIdOnlyInJM));
        assertThat(allocationsNotifiedFree.take(), is(allocationIdOnlyInTM));
        assertThat(failedSlotFutures.poll(5L, TimeUnit.MILLISECONDS), nullValue());
        assertThat(allocationsNotifiedFree.poll(5L, TimeUnit.MILLISECONDS), nullValue());
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}

Example 5 with AllocatedSlotReport

use of org.apache.flink.runtime.jobmaster.AllocatedSlotReport in project flink by apache.

the class TaskExecutorExecutionDeploymentReconciliationTest method testDeployedExecutionReporting.

@Test
public void testDeployedExecutionReporting() throws Exception {
    final OneShotLatch slotOfferLatch = new OneShotLatch();
    final BlockingQueue<Set<ExecutionAttemptID>> deployedExecutionsQueue = new ArrayBlockingQueue<>(3);
    final CompletableFuture<Void> taskFinishedFuture = new CompletableFuture<>();
    final ResourceID jobManagerResourceId = ResourceID.generate();
    final TestingJobMasterGateway jobMasterGateway = setupJobManagerGateway(slotOfferLatch, deployedExecutionsQueue, taskFinishedFuture, jobManagerResourceId);
    final CompletableFuture<SlotReport> initialSlotReportFuture = new CompletableFuture<>();
    final TestingResourceManagerGateway testingResourceManagerGateway = setupResourceManagerGateway(initialSlotReportFuture);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1, timeout)).setShuffleEnvironment(new NettyShuffleEnvironmentBuilder().build()).build();
    final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices);
    try {
        taskExecutor.start();
        taskExecutor.waitUntilStarted();
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        final TaskDeploymentDescriptor taskDeploymentDescriptor = createTaskDeploymentDescriptor(jobId);
        connectComponentsAndRequestSlot(jobMasterGateway, testingResourceManagerGateway, taskExecutorGateway, taskManagerServices.getJobLeaderService(), initialSlotReportFuture, taskDeploymentDescriptor.getAllocationId());
        TestingInvokable.sync = new BlockerSync();
        // This ensures TM has been successfully registered to JM.
        slotOfferLatch.await();
        AllocatedSlotReport slotAllocationReport = new AllocatedSlotReport(jobId, Collections.singleton(new AllocatedSlotInfo(0, taskDeploymentDescriptor.getAllocationId())));
        // nothing as deployed, so the deployment report should be empty
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasSize(0));
        taskExecutorGateway.submitTask(taskDeploymentDescriptor, jobMasterGateway.getFencingToken(), timeout).get();
        TestingInvokable.sync.awaitBlocker();
        // task is deployed, so the deployment report should contain it
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasItem(taskDeploymentDescriptor.getExecutionAttemptId()));
        TestingInvokable.sync.releaseBlocker();
        // task is finished ans was cleaned up, so the deployment report should be empty
        taskFinishedFuture.get();
        taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, slotAllocationReport);
        assertThat(deployedExecutionsQueue.take(), hasSize(0));
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}