Examples with ComponentMainThreadExecutorServiceAdapter org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter used on opensource projects

Example 1 with ComponentMainThreadExecutorServiceAdapter

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter in project flink by apache.

the class OperatorCoordinatorSchedulerTest method setupTestJobAndScheduler.

private DefaultScheduler setupTestJobAndScheduler(OperatorCoordinator.Provider provider, @Nullable TaskExecutorOperatorEventGateway taskExecutorOperatorEventGateway, @Nullable Consumer<JobGraph> jobGraphPreProcessing, boolean restartAllOnFailover) throws Exception {
    final OperatorIDPair opIds = OperatorIDPair.of(new OperatorID(), provider.getOperatorId());
    final JobVertex vertex = new JobVertex("Vertex with OperatorCoordinator", testVertexId, Collections.singletonList(opIds));
    vertex.setInvokableClass(NoOpInvokable.class);
    vertex.addOperatorCoordinator(new SerializedValue<>(provider));
    vertex.setParallelism(2);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(vertex).build();
    SchedulerTestingUtils.enableCheckpointing(jobGraph);
    if (jobGraphPreProcessing != null) {
        jobGraphPreProcessing.accept(jobGraph);
    }
    final ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutorServiceAdapter((ScheduledExecutorService) executor, Thread.currentThread());
    final SchedulerTestingUtils.DefaultSchedulerBuilder schedulerBuilder = taskExecutorOperatorEventGateway == null ? SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor) : SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor, taskExecutorOperatorEventGateway);
    if (restartAllOnFailover) {
        schedulerBuilder.setFailoverStrategyFactory(new RestartAllFailoverStrategy.Factory());
    }
    final DefaultScheduler scheduler = schedulerBuilder.setFutureExecutor(executor).setDelayExecutor(executor).build();
    this.createdScheduler = scheduler;
    return scheduler;
}

Example 2 with ComponentMainThreadExecutorServiceAdapter

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter in project flink by apache.

the class OperatorCoordinatorHolderTest method checkpointEventValueAtomicity.

private void checkpointEventValueAtomicity(final Function<OperatorCoordinator.Context, OperatorCoordinator> coordinatorCtor) throws Exception {
    final ManuallyTriggeredScheduledExecutorService executor = new ManuallyTriggeredScheduledExecutorService();
    final ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutorServiceAdapter((ScheduledExecutorService) executor, Thread.currentThread());
    final EventReceivingTasks sender = EventReceivingTasks.createForRunningTasks();
    final OperatorCoordinatorHolder holder = createCoordinatorHolder(sender, coordinatorCtor, mainThreadExecutor);
    // give the coordinator some time to emit some events. This isn't strictly necessary,
    // but it randomly alters the timings between the coordinator's thread (event sender) and
    // the main thread (holder). This should produce a flaky test if we missed some corner
    // cases.
    Thread.sleep(new Random().nextInt(10));
    executor.triggerAll();
    // trigger the checkpoint - this should also shut the valve as soon as the future is
    // completed
    final CompletableFuture<byte[]> checkpointFuture = new CompletableFuture<>();
    holder.checkpointCoordinator(0L, checkpointFuture);
    executor.triggerAll();
    // give the coordinator some time to emit some events. Same as above, this adds some
    // randomization
    Thread.sleep(new Random().nextInt(10));
    holder.close();
    executor.triggerAll();
    assertTrue(checkpointFuture.isDone());
    final int checkpointedNumber = bytesToInt(checkpointFuture.get());
    assertEquals(checkpointedNumber, sender.getNumberOfSentEvents());
    for (int i = 0; i < checkpointedNumber; i++) {
        assertEquals(i, ((TestOperatorEvent) sender.getAllSentEvents().get(i).event).getValue());
    }
}

Example 3 with ComponentMainThreadExecutorServiceAdapter

use of org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter in project flink by apache.

the class OperatorCoordinatorHolderTest method testCheckpointFailsIfSendingEventFailedBeforeTrigger.

@Test
public void testCheckpointFailsIfSendingEventFailedBeforeTrigger() throws Exception {
    final ReorderableManualExecutorService executor = new ReorderableManualExecutorService();
    final ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutorServiceAdapter((ScheduledExecutorService) executor, Thread.currentThread());
    CompletableFuture<Acknowledge> eventSendingResult = new CompletableFuture<>();
    final EventReceivingTasks tasks = EventReceivingTasks.createForRunningTasksWithRpcResult(eventSendingResult);
    final OperatorCoordinatorHolder holder = createCoordinatorHolder(tasks, TestingOperatorCoordinator::new, mainThreadExecutor);
    // Send one event without finishing it.
    getCoordinator(holder).getSubtaskGateway(0).sendEvent(new TestOperatorEvent(0));
    executor.triggerAll();
    // Finish the event sending. This will insert one runnable that handles
    // failed events to the executor. And we delay this runnable to
    // simulates checkpoints triggered before the failure get processed.
    executor.setDelayNewRunnables(true);
    eventSendingResult.completeExceptionally(new RuntimeException("Artificial"));
    executor.setDelayNewRunnables(false);
    // Trigger one checkpoint, the checkpoint should not be confirmed
    // before the failure get triggered.
    CompletableFuture<byte[]> checkpointResult = new CompletableFuture<>();
    holder.checkpointCoordinator(1, checkpointResult);
    executor.triggerAll();
    getCoordinator(holder).getLastTriggeredCheckpoint().complete(new byte[0]);
    executor.triggerAll();
    assertFalse(checkpointResult.isDone());
    // Then the failure finally get processed by fail the corresponding tasks.
    executor.executeAllDelayedRunnables();
    executor.triggerAll();
    // The checkpoint would be finally confirmed.
    assertTrue(checkpointResult.isCompletedExceptionally());
}