Examples with ComponentMainThreadExecutor org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor used on opensource projects

Example 1 with ComponentMainThreadExecutor

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor in project flink by apache.

the class Execution method releaseAssignedResource.

/**
 * Releases the assigned resource and completes the release future once the assigned resource
 * has been successfully released.
 *
 * @param cause for the resource release, null if none
 */
private void releaseAssignedResource(@Nullable Throwable cause) {
    assertRunningInJobMasterMainThread();
    final LogicalSlot slot = assignedResource;
    if (slot != null) {
        ComponentMainThreadExecutor jobMasterMainThreadExecutor = getVertex().getExecutionGraphAccessor().getJobMasterMainThreadExecutor();
        slot.releaseSlot(cause).whenComplete((Object ignored, Throwable throwable) -> {
            jobMasterMainThreadExecutor.assertRunningInMainThread();
            if (throwable != null) {
                releaseFuture.completeExceptionally(throwable);
            } else {
                releaseFuture.complete(null);
            }
        });
    } else {
        // no assigned resource --> we can directly complete the release future
        releaseFuture.complete(null);
    }
}

Example 2 with ComponentMainThreadExecutor

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor in project flink by apache.

the class AdaptiveScheduler method tryToAssignSlots.

@Override
public CreatingExecutionGraph.AssignmentResult tryToAssignSlots(CreatingExecutionGraph.ExecutionGraphWithVertexParallelism executionGraphWithVertexParallelism) {
    final ExecutionGraph executionGraph = executionGraphWithVertexParallelism.getExecutionGraph();
    executionGraph.start(componentMainThreadExecutor);
    executionGraph.transitionToRunning();
    executionGraph.setInternalTaskFailuresListener(new UpdateSchedulerNgOnInternalFailuresListener(this));
    final VertexParallelism vertexParallelism = executionGraphWithVertexParallelism.getVertexParallelism();
    return slotAllocator.tryReserveResources(vertexParallelism).map(reservedSlots -> CreatingExecutionGraph.AssignmentResult.success(assignSlotsToExecutionGraph(executionGraph, reservedSlots))).orElseGet(CreatingExecutionGraph.AssignmentResult::notPossible);
}

Example 3 with ComponentMainThreadExecutor

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor in project flink by apache.

the class SchedulerBenchmarkUtils method createAndInitExecutionGraph.

public static ExecutionGraph createAndInitExecutionGraph(List<JobVertex> jobVertices, JobConfiguration jobConfiguration, ScheduledExecutorService scheduledExecutorService) throws Exception {
    final JobGraph jobGraph = createJobGraph(jobVertices, jobConfiguration);
    final ComponentMainThreadExecutor mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
    final DefaultScheduler scheduler = SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor).setIoExecutor(scheduledExecutorService).setFutureExecutor(scheduledExecutorService).setDelayExecutor(new ScheduledExecutorServiceAdapter(scheduledExecutorService)).build();
    return scheduler.getExecutionGraph();
}

Example 4 with ComponentMainThreadExecutor

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor in project flink by apache.

the class DeclarativeSlotPoolBridgeResourceDeclarationTest method testRequirementsDecreasedOnAllocationTimeout.

@Test
public void testRequirementsDecreasedOnAllocationTimeout() throws Exception {
    final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
    try {
        ComponentMainThreadExecutor mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forSingleThreadExecutor(scheduledExecutorService);
        declarativeSlotPoolBridge.start(jobMasterId, "localhost", mainThreadExecutor);
        // requesting the allocation of a new slot increases the requirements
        final CompletableFuture<PhysicalSlot> allocationFuture = CompletableFuture.supplyAsync(() -> declarativeSlotPoolBridge.requestNewAllocatedSlot(new SlotRequestId(), ResourceProfile.UNKNOWN, Time.milliseconds(5)), mainThreadExecutor).get();
        // waiting for the timeout
        assertThat(allocationFuture, FlinkMatchers.futureWillCompleteExceptionally(Duration.ofMinutes(1)));
        // when the allocation fails the requirements should be reduced (it is the users
        // responsibility to retry)
        CompletableFuture.runAsync(() -> assertThat(requirementListener.getRequirements().getResourceCount(ResourceProfile.UNKNOWN), is(0)), mainThreadExecutor).join();
    } finally {
        scheduledExecutorService.shutdown();
    }
}

Example 5 with ComponentMainThreadExecutor

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor in project flink by apache.

the class DefaultSchedulerTest method testStatusMetrics.

@Test
public void testStatusMetrics() throws Exception {
    // running time acts as a stand-in for generic status time metrics
    final CompletableFuture<Gauge<Long>> runningTimeMetricFuture = new CompletableFuture<>();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((metric, name, group) -> {
        switch(name) {
            case "runningTimeTotal":
                runningTimeMetricFuture.complete((Gauge<Long>) metric);
                break;
        }
    }).build();
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph);
    final Configuration configuration = new Configuration();
    configuration.set(MetricOptions.JOB_STATUS_METRICS, Arrays.asList(MetricOptions.JobStatusMetrics.TOTAL_TIME));
    final ComponentMainThreadExecutor singleThreadMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forSingleThreadExecutor(scheduledExecutorService);
    final Time slotTimeout = Time.milliseconds(5L);
    final SlotPool slotPool = new DeclarativeSlotPoolBridgeBuilder().setBatchSlotTimeout(slotTimeout).buildAndStart(singleThreadMainThreadExecutor);
    final PhysicalSlotProvider slotProvider = new PhysicalSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
    final DefaultScheduler scheduler = createSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobManagerJobMetricGroup(JobManagerMetricGroup.createJobManagerMetricGroup(metricRegistry, "localhost").addJob(new JobID(), "jobName")).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(slotProvider, slotTimeout)).build();
    final AdaptiveSchedulerTest.SubmissionBufferingTaskManagerGateway taskManagerGateway = new AdaptiveSchedulerTest.SubmissionBufferingTaskManagerGateway(1);
    taskManagerGateway.setCancelConsumer(executionAttemptId -> {
        singleThreadMainThreadExecutor.execute(() -> scheduler.updateTaskExecutionState(new TaskExecutionState(executionAttemptId, ExecutionState.CANCELED)));
    });
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        offerSlots(slotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // wait for the first task submission
    taskManagerGateway.waitForSubmissions(1, Duration.ofSeconds(5));
    // sleep a bit to ensure uptime is > 0
    Thread.sleep(10L);
    final Gauge<Long> runningTimeGauge = runningTimeMetricFuture.get();
    Assert.assertThat(runningTimeGauge.getValue(), greaterThan(0L));
}