Examples with TaskExecutorGateway org.apache.flink.runtime.taskexecutor.TaskExecutorGateway used on opensource projects

Example 36 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testRequirementDeclaration.

private void testRequirementDeclaration(RequirementDeclarationScenario scenario) throws Exception {
    final ResourceID resourceID = ResourceID.generate();
    final JobID jobId = new JobID();
    final SlotID slotId = SlotID.getDynamicSlotID(resourceID);
    final String targetAddress = "localhost";
    final ResourceRequirements requirements = ResourceRequirements.create(jobId, targetAddress, Collections.singleton(ResourceRequirement.create(DEFAULT_SLOT_RESOURCE_PROFILE, 1)));
    final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
    // accept an incoming slot request
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        requestFuture.complete(tuple6);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    new Context() {

        {
            runTest(() -> {
                if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_BEFORE_REQUIREMENT_DECLARATION) {
                    runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                }
                runInMainThread(() -> getSlotManager().processResourceRequirements(requirements));
                if (scenario == RequirementDeclarationScenario.TASK_EXECUTOR_REGISTRATION_AFTER_REQUIREMENT_DECLARATION) {
                    runInMainThread(() -> getSlotManager().registerTaskManager(taskExecutorConnection, new SlotReport(), DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE));
                }
                assertThat(assertFutureCompleteAndReturn(requestFuture), is(equalTo(Tuple6.of(slotId, jobId, assertFutureCompleteAndReturn(requestFuture).f2, DEFAULT_SLOT_RESOURCE_PROFILE, targetAddress, getResourceManagerId()))));
                final TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(requestFuture).f2).get();
                assertEquals("The slot has not been allocated to the expected allocation id.", assertFutureCompleteAndReturn(requestFuture).f2, slot.getAllocationId());
            });
        }
    };
}

Example 37 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testResourceCanBeAllocatedForDifferentJobAfterFree.

/**
 * Tests that a resource allocated for one job can be allocated for another job after being
 * freed.
 */
private void testResourceCanBeAllocatedForDifferentJobAfterFree(SecondRequirementDeclarationTime secondRequirementDeclarationTime) throws Exception {
    final CompletableFuture<AllocationID> allocationIdFuture1 = new CompletableFuture<>();
    final CompletableFuture<AllocationID> allocationIdFuture2 = new CompletableFuture<>();
    final ResourceRequirements resourceRequirements1 = createResourceRequirementsForSingleSlot();
    final ResourceRequirements resourceRequirements2 = createResourceRequirementsForSingleSlot();
    final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        if (!allocationIdFuture1.isDone()) {
            allocationIdFuture1.complete(tuple6.f2);
        } else {
            allocationIdFuture2.complete(tuple6.f2);
        }
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    final ResourceID resourceID = ResourceID.generate();
    final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().registerTaskManager(taskManagerConnection, slotReport, DEFAULT_SLOT_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                    getSlotManager().processResourceRequirements(resourceRequirements1);
                });
                final AllocationID allocationId1 = assertFutureCompleteAndReturn(allocationIdFuture1);
                TaskManagerSlotInformation slot = getTaskManagerTracker().getAllocatedOrPendingSlot(allocationId1).get();
                assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements1.getJobId(), slot.getJobId());
                if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.BEFORE_FREE) {
                    runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
                }
                // clear resource requirements first so that the freed slot isn't
                // immediately re-assigned to the job
                runInMainThread(() -> {
                    getSlotManager().processResourceRequirements(ResourceRequirements.create(resourceRequirements1.getJobId(), resourceRequirements1.getTargetAddress(), Collections.emptyList()));
                    getSlotManager().freeSlot(SlotID.getDynamicSlotID(resourceID), allocationId1);
                });
                if (secondRequirementDeclarationTime == SecondRequirementDeclarationTime.AFTER_FREE) {
                    runInMainThread(() -> getSlotManager().processResourceRequirements(resourceRequirements2));
                }
                slot = getTaskManagerTracker().getAllocatedOrPendingSlot(assertFutureCompleteAndReturn(allocationIdFuture2)).get();
                assertEquals("The slot has not been allocated to the expected job id.", resourceRequirements2.getJobId(), slot.getJobId());
            });
        }
    };
}

Example 38 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class AbstractFineGrainedSlotManagerITCase method testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsAllocatedAfterSlotReport() throws Exception {
    final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(tuple6 -> {
        allocationIdFuture.complete(tuple6.f2);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).createTestingTaskExecutorGateway();
    // The fatal error handler will exit the system if there is any exceptions in handling the
    // ack of request slot. We need the security manager to verify that would not happen.
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    final TaskExecutorConnection taskExecutionConnection = new TaskExecutorConnection(ResourceID.generate(), taskExecutorGateway);
    final SlotReport slotReport = new SlotReport();
    new Context() {

        {
            runTest(() -> {
                runInMainThread(() -> {
                    getSlotManager().processResourceRequirements(createResourceRequirements(new JobID(), 1));
                    getSlotManager().registerTaskManager(taskExecutionConnection, slotReport, DEFAULT_TOTAL_RESOURCE_PROFILE, DEFAULT_SLOT_RESOURCE_PROFILE);
                });
                final AllocationID allocationId = assertFutureCompleteAndReturn(allocationIdFuture);
                runInMainThread(() -> getSlotManager().reportSlotStatus(taskExecutionConnection.getInstanceID(), new SlotReport(createAllocatedSlotStatus(allocationId, DEFAULT_SLOT_RESOURCE_PROFILE))));
                assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
            });
        }
    };
    System.setSecurityManager(null);
}

Example 39 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob.

@Test
public void testAllocationUpdatesIgnoredIfSlotMarkedAsPendingForOtherJob() throws Exception {
    final DefaultSlotTracker slotTracker = new DefaultSlotTracker();
    final CompletableFuture<AllocationID> firstSlotAllocationIdFuture = new CompletableFuture<>();
    final CompletableFuture<Acknowledge> firstSlotRequestAcknowledgeFuture = new CompletableFuture<>();
    final Iterator<CompletableFuture<Acknowledge>> slotRequestAcknowledgeFutures = Arrays.asList(firstSlotRequestAcknowledgeFuture, new CompletableFuture<Acknowledge>()).iterator();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(requestSlotParameters -> {
        firstSlotAllocationIdFuture.complete(requestSlotParameters.f2);
        return slotRequestAcknowledgeFutures.next();
    }).createTestingTaskExecutorGateway();
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotTracker(slotTracker).buildAndStart(ResourceManagerId.generate(), ComponentMainThreadExecutorServiceAdapter.forMainThread(), new TestingResourceActionsBuilder().build())) {
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        final SlotID slotId = Iterators.getOnlyElement(slotReport.iterator()).getSlotID();
        // register task executor
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.reportSlotStatus(taskExecutionConnection.getInstanceID(), createSlotReport(taskExecutionConnection.getResourceID(), 1));
        // triggers the allocation of a slot
        final JobID firstJobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(firstJobId, 1));
        // clear requirements immediately to ensure the slot will not get re-allocated to the
        // same job
        slotManager.processResourceRequirements(ResourceRequirements.empty(firstJobId, "foobar"));
        // when the slot is freed it will be re-assigned to this second job
        slotManager.processResourceRequirements(createResourceRequirements(new JobID(), 1));
        slotManager.freeSlot(slotId, firstSlotAllocationIdFuture.get());
        // acknowledge the first allocation
        // this should fail if the acknowledgement is not ignored
        firstSlotRequestAcknowledgeFuture.complete(Acknowledge.get());
        // sanity check that the acknowledge was really ignored
        assertThat(slotTracker.getSlot(slotId).getJobId(), is(not(firstJobId)));
    }
}

Example 40 with TaskExecutorGateway

use of org.apache.flink.runtime.taskexecutor.TaskExecutorGateway in project flink by apache.

the class DeclarativeSlotManagerTest method testAllocationUpdatesIgnoredIfTaskExecutorUnregistered.

@Test
public void testAllocationUpdatesIgnoredIfTaskExecutorUnregistered() throws Exception {
    final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
    final ResourceTracker resourceTracker = new DefaultResourceTracker();
    final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setRequestSlotFunction(ignored -> CompletableFuture.completedFuture(Acknowledge.get())).createTestingTaskExecutorGateway();
    final SystemExitTrackingSecurityManager trackingSecurityManager = new SystemExitTrackingSecurityManager();
    System.setSecurityManager(trackingSecurityManager);
    try (final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setResourceTracker(resourceTracker).buildAndStart(ResourceManagerId.generate(), executor, new TestingResourceActionsBuilder().build())) {
        JobID jobId = new JobID();
        slotManager.processResourceRequirements(createResourceRequirements(jobId, 1));
        final TaskExecutorConnection taskExecutionConnection = createTaskExecutorConnection(taskExecutorGateway);
        final SlotReport slotReport = createSlotReport(taskExecutionConnection.getResourceID(), 1);
        slotManager.registerTaskManager(taskExecutionConnection, slotReport, ResourceProfile.ANY, ResourceProfile.ANY);
        slotManager.unregisterTaskManager(taskExecutionConnection.getInstanceID(), TEST_EXCEPTION);
        executor.triggerAll();
        assertThat(trackingSecurityManager.getSystemExitFuture().isDone(), is(false));
    } finally {
        System.setSecurityManager(null);
    }
}