Examples with ScheduledExecutor org.apache.flink.runtime.concurrent.ScheduledExecutor used on opensource projects

Example 6 with ScheduledExecutor

use of org.apache.flink.runtime.concurrent.ScheduledExecutor in project flink by apache.

the class JobLeaderIdServiceTest method testInitialJobTimeout.

/**
	 * Tests that the initial job registration registers a timeout which will call
	 * {@link JobLeaderIdActions#notifyJobTimeout(JobID, UUID)} when executed.
	 */
@Test
public void testInitialJobTimeout() throws Exception {
    final JobID jobId = new JobID();
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    Time timeout = Time.milliseconds(5000L);
    JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices, scheduledExecutor, timeout);
    jobLeaderIdService.start(jobLeaderIdActions);
    jobLeaderIdService.addJob(jobId);
    assertTrue(jobLeaderIdService.containsJob(jobId));
    ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
    verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), anyLong(), any(TimeUnit.class));
    Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
    timeoutRunnable.run();
    ArgumentCaptor<UUID> timeoutIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
    verify(jobLeaderIdActions, times(1)).notifyJobTimeout(eq(jobId), timeoutIdArgumentCaptor.capture());
    assertTrue(jobLeaderIdService.isValidTimeout(jobId, timeoutIdArgumentCaptor.getValue()));
}

Example 7 with ScheduledExecutor

use of org.apache.flink.runtime.concurrent.ScheduledExecutor in project flink by apache.

the class JobLeaderIdServiceTest method testRemovingJob.

/**
	 * Tests that removing a job completes the job leader id future exceptionally
	 */
@Test(timeout = 10000)
public void testRemovingJob() throws Exception {
    final JobID jobId = new JobID();
    TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    Time timeout = Time.milliseconds(5000L);
    JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);
    JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices, scheduledExecutor, timeout);
    jobLeaderIdService.start(jobLeaderIdActions);
    jobLeaderIdService.addJob(jobId);
    Future<UUID> leaderIdFuture = jobLeaderIdService.getLeaderId(jobId);
    // remove the job before we could find a leader
    jobLeaderIdService.removeJob(jobId);
    assertFalse(jobLeaderIdService.containsJob(jobId));
    try {
        leaderIdFuture.get();
        fail("The leader id future should be completed exceptionally.");
    } catch (ExecutionException ignored) {
    // expected exception
    }
}

Example 8 with ScheduledExecutor

use of org.apache.flink.runtime.concurrent.ScheduledExecutor in project flink by apache.

the class AkkaRpcServiceTest method testScheduledExecutorServiceWithFixedDelaySchedule.

/**
	 * Tests that the RPC service's scheduled executor service can execute runnable with a fixed
	 * delay.
	 */
@Test(timeout = 60000)
public void testScheduledExecutorServiceWithFixedDelaySchedule() throws ExecutionException, InterruptedException {
    ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();
    final int tries = 4;
    final long delay = 10L;
    final CountDownLatch countDownLatch = new CountDownLatch(tries);
    long currentTime = System.nanoTime();
    ScheduledFuture<?> future = scheduledExecutor.scheduleWithFixedDelay(new Runnable() {

        @Override
        public void run() {
            countDownLatch.countDown();
        }
    }, delay, delay, TimeUnit.MILLISECONDS);
    assertTrue(!future.isDone());
    countDownLatch.await();
    // the future should not complete since we have a periodic task
    assertTrue(!future.isDone());
    long finalTime = System.nanoTime() - currentTime;
    // the processing should have taken at least delay times the number of count downs.
    assertTrue(finalTime >= tries * delay);
    future.cancel(true);
}

Example 9 with ScheduledExecutor

use of org.apache.flink.runtime.concurrent.ScheduledExecutor in project flink by apache.

the class AkkaRpcServiceTest method testScheduledExecutorServiceCancelWithFixedDelay.

/**
	 * Tests that canceling the returned future will stop the execution of the scheduled runnable.
	 */
@Test
public void testScheduledExecutorServiceCancelWithFixedDelay() throws InterruptedException {
    ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();
    long delay = 10L;
    final OneShotLatch futureTask = new OneShotLatch();
    final OneShotLatch latch = new OneShotLatch();
    final OneShotLatch shouldNotBeTriggeredLatch = new OneShotLatch();
    ScheduledFuture<?> future = scheduledExecutor.scheduleWithFixedDelay(new Runnable() {

        @Override
        public void run() {
            try {
                if (!futureTask.isTriggered()) {
                    // first run
                    futureTask.trigger();
                    latch.await();
                } else {
                    shouldNotBeTriggeredLatch.trigger();
                }
            } catch (InterruptedException e) {
            // ignore
            }
        }
    }, delay, delay, TimeUnit.MILLISECONDS);
    // wait until we're in the runnable
    futureTask.await();
    // cancel the scheduled future
    future.cancel(false);
    latch.trigger();
    try {
        shouldNotBeTriggeredLatch.await(5 * delay, TimeUnit.MILLISECONDS);
        fail("The shouldNotBeTriggeredLatch should never be triggered.");
    } catch (TimeoutException e) {
    // expected
    }
}

Example 10 with ScheduledExecutor

use of org.apache.flink.runtime.concurrent.ScheduledExecutor in project flink by apache.

the class TaskExecutorTest method testHeartbeatTimeoutWithJobManager.

@Test
public void testHeartbeatTimeoutWithJobManager() throws Exception {
    final JobID jobId = new JobID();
    final Configuration configuration = new Configuration();
    final TaskManagerConfiguration tmConfig = TaskManagerConfiguration.fromConfiguration(configuration);
    final ResourceID tmResourceId = new ResourceID("tm");
    final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(tmResourceId, InetAddress.getLoopbackAddress(), 1234);
    final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class)), mock(TimerService.class));
    final TestingSerialRpcService rpc = new TestingSerialRpcService();
    final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
    final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
    final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService();
    final TestingLeaderRetrievalService jmLeaderRetrievalService = new TestingLeaderRetrievalService();
    haServices.setJobMasterLeaderRetriever(jobId, jmLeaderRetrievalService);
    haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
    final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
    final long heartbeatTimeout = 10L;
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    when(heartbeatServices.createHeartbeatManager(eq(taskManagerLocation.getResourceID()), any(HeartbeatListener.class), any(ScheduledExecutor.class), any(Logger.class))).thenAnswer(new Answer<HeartbeatManagerImpl<Void, Void>>() {

        @Override
        public HeartbeatManagerImpl<Void, Void> answer(InvocationOnMock invocation) throws Throwable {
            return new HeartbeatManagerImpl<>(heartbeatTimeout, taskManagerLocation.getResourceID(), (HeartbeatListener<Void, Void>) invocation.getArguments()[1], (Executor) invocation.getArguments()[2], (ScheduledExecutor) invocation.getArguments()[2], (Logger) invocation.getArguments()[3]);
        }
    });
    final String jobMasterAddress = "jm";
    final UUID jmLeaderId = UUID.randomUUID();
    final ResourceID jmResourceId = new ResourceID(jobMasterAddress);
    final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
    final int blobPort = 42;
    when(jobMasterGateway.registerTaskManager(any(String.class), eq(taskManagerLocation), eq(jmLeaderId), any(Time.class))).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
    when(jobMasterGateway.getAddress()).thenReturn(jobMasterAddress);
    when(jobMasterGateway.getHostname()).thenReturn("localhost");
    try {
        final TaskExecutor taskManager = new TaskExecutor(tmConfig, taskManagerLocation, rpc, mock(MemoryManager.class), mock(IOManager.class), mock(NetworkEnvironment.class), haServices, heartbeatServices, mock(MetricRegistry.class), mock(TaskManagerMetricGroup.class), mock(BroadcastVariableManager.class), mock(FileCache.class), taskSlotTable, new JobManagerTable(), jobLeaderService, testingFatalErrorHandler);
        taskManager.start();
        rpc.registerGateway(jobMasterAddress, jobMasterGateway);
        // we have to add the job after the TaskExecutor, because otherwise the service has not
        // been properly started.
        jobLeaderService.addJob(jobId, jobMasterAddress);
        // now inform the task manager about the new job leader
        jmLeaderRetrievalService.notifyListener(jobMasterAddress, jmLeaderId);
        // register task manager success will trigger monitoring heartbeat target between tm and jm
        verify(jobMasterGateway).registerTaskManager(eq(taskManager.getAddress()), eq(taskManagerLocation), eq(jmLeaderId), any(Time.class));
        // the timeout should trigger disconnecting from the JobManager
        verify(jobMasterGateway, timeout(heartbeatTimeout * 5)).disconnectTaskManager(eq(taskManagerLocation.getResourceID()), any(TimeoutException.class));
        // check if a concurrent error occurred
        testingFatalErrorHandler.rethrowError();
    } finally {
        rpc.stopService();
    }
}