Examples with ScheduledExecutorServiceAdapter org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter used on opensource projects

Example 6 with ScheduledExecutorServiceAdapter

use of org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter in project flink by apache.

the class AdaptiveBatchSchedulerFactory method createInstance.

@Override
public SchedulerNG createInstance(Logger log, JobGraph jobGraph, Executor ioExecutor, Configuration jobMasterConfiguration, SlotPoolService slotPoolService, ScheduledExecutorService futureExecutor, ClassLoader userCodeLoader, CheckpointRecoveryFactory checkpointRecoveryFactory, Time rpcTimeout, BlobWriter blobWriter, JobManagerJobMetricGroup jobManagerJobMetricGroup, Time slotRequestTimeout, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionDeploymentTracker executionDeploymentTracker, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, FatalErrorHandler fatalErrorHandler, JobStatusListener jobStatusListener) throws Exception {
    checkState(jobGraph.getJobType() == JobType.BATCH, "Adaptive batch scheduler only supports batch jobs");
    checkAllExchangesBlocking(jobGraph);
    final SlotPool slotPool = slotPoolService.castInto(SlotPool.class).orElseThrow(() -> new IllegalStateException("The DefaultScheduler requires a SlotPool."));
    final SlotSelectionStrategy slotSelectionStrategy = SlotSelectionStrategyUtils.selectSlotSelectionStrategy(JobType.BATCH, jobMasterConfiguration);
    final PhysicalSlotRequestBulkChecker bulkChecker = PhysicalSlotRequestBulkCheckerImpl.createFromSlotPool(slotPool, SystemClock.getInstance());
    final PhysicalSlotProvider physicalSlotProvider = new PhysicalSlotProviderImpl(slotSelectionStrategy, slotPool);
    final ExecutionSlotAllocatorFactory allocatorFactory = new SlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider, false, bulkChecker, slotRequestTimeout);
    final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader.createRestartBackoffTimeStrategyFactory(jobGraph.getSerializedExecutionConfig().deserializeValue(userCodeLoader).getRestartStrategy(), jobMasterConfiguration, jobGraph.isCheckpointingEnabled()).create();
    log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());
    final ExecutionGraphFactory executionGraphFactory = new DefaultExecutionGraphFactory(jobMasterConfiguration, userCodeLoader, executionDeploymentTracker, futureExecutor, ioExecutor, rpcTimeout, jobManagerJobMetricGroup, blobWriter, shuffleMaster, partitionTracker, true);
    return new AdaptiveBatchScheduler(log, jobGraph, ioExecutor, jobMasterConfiguration, bulkChecker::start, new ScheduledExecutorServiceAdapter(futureExecutor), userCodeLoader, new CheckpointsCleaner(), checkpointRecoveryFactory, jobManagerJobMetricGroup, new VertexwiseSchedulingStrategy.Factory(), FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration), restartBackoffTimeStrategy, new DefaultExecutionVertexOperations(), new ExecutionVertexVersioner(), allocatorFactory, initializationTimestamp, mainThreadExecutor, jobStatusListener, executionGraphFactory, shuffleMaster, rpcTimeout, DefaultVertexParallelismDecider.from(jobMasterConfiguration), jobMasterConfiguration.getInteger(JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MAX_PARALLELISM));
}

Example 7 with ScheduledExecutorServiceAdapter

use of org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter in project flink by apache.

the class HeartbeatManagerTest method testHeartbeatManagerSenderTargetPayload.

/**
 * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link
 * HeartbeatListener} by the {@link HeartbeatManagerSenderImpl}.
 */
@Test
public void testHeartbeatManagerSenderTargetPayload() throws Exception {
    final long heartbeatTimeout = 100L;
    final long heartbeatPeriod = 2000L;
    final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1);
    final ResourceID someTargetId = ResourceID.generate();
    final ResourceID specialTargetId = ResourceID.generate();
    final OneShotLatch someTargetReceivedLatch = new OneShotLatch();
    final OneShotLatch specialTargetReceivedLatch = new OneShotLatch();
    final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(someTargetReceivedLatch);
    final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(specialTargetReceivedLatch);
    final int defaultResponse = 0;
    final int specialResponse = 1;
    HeartbeatManager<?, Integer> heartbeatManager = new HeartbeatManagerSenderImpl<>(heartbeatPeriod, heartbeatTimeout, FAILED_RPC_THRESHOLD, ResourceID.generate(), new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse), new ScheduledExecutorServiceAdapter(scheduledThreadPoolExecutor), LOG);
    try {
        heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget);
        heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget);
        someTargetReceivedLatch.await(5, TimeUnit.SECONDS);
        specialTargetReceivedLatch.await(5, TimeUnit.SECONDS);
        assertEquals(defaultResponse, someHeartbeatTarget.getLastRequestedHeartbeatPayload());
        assertEquals(specialResponse, specialHeartbeatTarget.getLastRequestedHeartbeatPayload());
    } finally {
        heartbeatManager.stop();
        scheduledThreadPoolExecutor.shutdown();
    }
}

Example 8 with ScheduledExecutorServiceAdapter

use of org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter in project flink by apache.

the class SchedulerBenchmarkUtils method createAndInitExecutionGraph.

public static ExecutionGraph createAndInitExecutionGraph(List<JobVertex> jobVertices, JobConfiguration jobConfiguration, ScheduledExecutorService scheduledExecutorService) throws Exception {
    final JobGraph jobGraph = createJobGraph(jobVertices, jobConfiguration);
    final ComponentMainThreadExecutor mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
    final DefaultScheduler scheduler = SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor).setIoExecutor(scheduledExecutorService).setFutureExecutor(scheduledExecutorService).setDelayExecutor(new ScheduledExecutorServiceAdapter(scheduledExecutorService)).build();
    return scheduler.getExecutionGraph();
}

Example 9 with ScheduledExecutorServiceAdapter

use of org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter in project flink by apache.

the class FlinkYarnSessionCli method run.

public int run(String[] args) throws CliArgsException, FlinkException {
    // 
    // Command Line Options
    // 
    final CommandLine cmd = parseCommandLineOptions(args, true);
    if (cmd.hasOption(help.getOpt())) {
        printUsage();
        return 0;
    }
    final Configuration effectiveConfiguration = new Configuration(configuration);
    final Configuration commandLineConfiguration = toConfiguration(cmd);
    effectiveConfiguration.addAll(commandLineConfiguration);
    LOG.debug("Effective configuration: {}", effectiveConfiguration);
    final ClusterClientFactory<ApplicationId> yarnClusterClientFactory = clusterClientServiceLoader.getClusterClientFactory(effectiveConfiguration);
    effectiveConfiguration.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName());
    final YarnClusterDescriptor yarnClusterDescriptor = (YarnClusterDescriptor) yarnClusterClientFactory.createClusterDescriptor(effectiveConfiguration);
    try {
        // Query cluster for metrics
        if (cmd.hasOption(query.getOpt())) {
            final String description = yarnClusterDescriptor.getClusterDescription();
            System.out.println(description);
            return 0;
        } else {
            final ClusterClientProvider<ApplicationId> clusterClientProvider;
            final ApplicationId yarnApplicationId;
            if (cmd.hasOption(applicationId.getOpt())) {
                yarnApplicationId = ConverterUtils.toApplicationId(cmd.getOptionValue(applicationId.getOpt()));
                clusterClientProvider = yarnClusterDescriptor.retrieve(yarnApplicationId);
            } else {
                final ClusterSpecification clusterSpecification = yarnClusterClientFactory.getClusterSpecification(effectiveConfiguration);
                clusterClientProvider = yarnClusterDescriptor.deploySessionCluster(clusterSpecification);
                ClusterClient<ApplicationId> clusterClient = clusterClientProvider.getClusterClient();
                // ------------------ ClusterClient deployed, handle connection details
                yarnApplicationId = clusterClient.getClusterId();
                try {
                    System.out.println("JobManager Web Interface: " + clusterClient.getWebInterfaceURL());
                    writeYarnPropertiesFile(yarnApplicationId, dynamicPropertiesEncoded);
                } catch (Exception e) {
                    try {
                        clusterClient.close();
                    } catch (Exception ex) {
                        LOG.info("Could not properly shutdown cluster client.", ex);
                    }
                    try {
                        yarnClusterDescriptor.killCluster(yarnApplicationId);
                    } catch (FlinkException fe) {
                        LOG.info("Could not properly terminate the Flink cluster.", fe);
                    }
                    throw new FlinkException("Could not write the Yarn connection information.", e);
                }
            }
            if (!effectiveConfiguration.getBoolean(DeploymentOptions.ATTACHED)) {
                YarnClusterDescriptor.logDetachedClusterInformation(yarnApplicationId, LOG);
            } else {
                ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
                final YarnApplicationStatusMonitor yarnApplicationStatusMonitor = new YarnApplicationStatusMonitor(yarnClusterDescriptor.getYarnClient(), yarnApplicationId, new ScheduledExecutorServiceAdapter(scheduledExecutorService));
                Thread shutdownHook = ShutdownHookUtil.addShutdownHook(() -> shutdownCluster(clusterClientProvider.getClusterClient(), scheduledExecutorService, yarnApplicationStatusMonitor), getClass().getSimpleName(), LOG);
                try {
                    runInteractiveCli(yarnApplicationStatusMonitor, acceptInteractiveInput);
                } finally {
                    shutdownCluster(clusterClientProvider.getClusterClient(), scheduledExecutorService, yarnApplicationStatusMonitor);
                    if (shutdownHook != null) {
                        // we do not need the hook anymore as we have just tried to shutdown the
                        // cluster.
                        ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);
                    }
                    tryRetrieveAndLogApplicationReport(yarnClusterDescriptor.getYarnClient(), yarnApplicationId);
                }
            }
        }
    } finally {
        try {
            yarnClusterDescriptor.close();
        } catch (Exception e) {
            LOG.info("Could not properly close the yarn cluster descriptor.", e);
        }
    }
    return 0;
}

Example 10 with ScheduledExecutorServiceAdapter

use of org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter in project flink by apache.

the class JobStatusPollingUtilsTest method testHappyPath.

@Test
public void testHappyPath() throws ExecutionException, InterruptedException {
    final int maxAttemptCounter = 1;
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    try {
        final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);
        final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);
        final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(jobStatusSupplier, () -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))), scheduledExecutor, 10);
        result.join();
        assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
        assertTrue(result.isDone() && result.get().isSuccess());
    } finally {
        ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
    }
}