Examples with PipelineExecutorServiceLoader org.apache.flink.core.execution.PipelineExecutorServiceLoader used on opensource projects

Example 1 with PipelineExecutorServiceLoader

use of org.apache.flink.core.execution.PipelineExecutorServiceLoader in project flink by apache.

the class StreamContextEnvironment method setAsContext.

// --------------------------------------------------------------------------------------------
public static void setAsContext(final PipelineExecutorServiceLoader executorServiceLoader, final Configuration configuration, final ClassLoader userCodeClassLoader, final boolean enforceSingleJobExecution, final boolean suppressSysout) {
    StreamExecutionEnvironmentFactory factory = conf -> {
        final List<String> errors = new ArrayList<>();
        final boolean allowConfigurations = configuration.getBoolean(DeploymentOptions.ALLOW_CLIENT_JOB_CONFIGURATIONS);
        if (!allowConfigurations && !conf.toMap().isEmpty()) {
            conf.toMap().forEach((k, v) -> errors.add(ConfigurationNotAllowedMessage.ofConfigurationKeyAndValue(k, v)));
        }
        Configuration mergedConfiguration = new Configuration();
        mergedConfiguration.addAll(configuration);
        mergedConfiguration.addAll(conf);
        return new StreamContextEnvironment(executorServiceLoader, mergedConfiguration, userCodeClassLoader, enforceSingleJobExecution, suppressSysout, allowConfigurations, errors);
    };
    initializeContextEnvironment(factory);
}

Example 2 with PipelineExecutorServiceLoader

use of org.apache.flink.core.execution.PipelineExecutorServiceLoader in project flink by apache.

the class ApplicationDispatcherBootstrap method runApplicationEntryPoint.

/**
 * Runs the user program entrypoint and completes the given {@code jobIdsFuture} with the {@link
 * JobID JobIDs} of the submitted jobs.
 *
 * <p>This should be executed in a separate thread (or task).
 */
private void runApplicationEntryPoint(final CompletableFuture<List<JobID>> jobIdsFuture, final Set<JobID> tolerateMissingResult, final DispatcherGateway dispatcherGateway, final ScheduledExecutor scheduledExecutor, final boolean enforceSingleJobExecution, final boolean submitFailedJobOnApplicationError) {
    if (submitFailedJobOnApplicationError && !enforceSingleJobExecution) {
        jobIdsFuture.completeExceptionally(new ApplicationExecutionException(String.format("Submission of failed job in case of an application error ('%s') is not supported in non-HA setups.", DeploymentOptions.SUBMIT_FAILED_JOB_ON_APPLICATION_ERROR.key())));
        return;
    }
    final List<JobID> applicationJobIds = new ArrayList<>(recoveredJobIds);
    try {
        final PipelineExecutorServiceLoader executorServiceLoader = new EmbeddedExecutorServiceLoader(applicationJobIds, dispatcherGateway, scheduledExecutor);
        ClientUtils.executeProgram(executorServiceLoader, configuration, application, enforceSingleJobExecution, true);
        if (applicationJobIds.isEmpty()) {
            jobIdsFuture.completeExceptionally(new ApplicationExecutionException("The application contains no execute() calls."));
        } else {
            jobIdsFuture.complete(applicationJobIds);
        }
    } catch (Throwable t) {
        // If we're running in a single job execution mode, it's safe to consider re-submission
        // of an already finished a success.
        final Optional<DuplicateJobSubmissionException> maybeDuplicate = ExceptionUtils.findThrowable(t, DuplicateJobSubmissionException.class);
        if (enforceSingleJobExecution && maybeDuplicate.isPresent() && maybeDuplicate.get().isGloballyTerminated()) {
            final JobID jobId = maybeDuplicate.get().getJobID();
            tolerateMissingResult.add(jobId);
            jobIdsFuture.complete(Collections.singletonList(jobId));
        } else if (submitFailedJobOnApplicationError && applicationJobIds.isEmpty()) {
            final JobID failedJobId = JobID.fromHexString(configuration.get(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID));
            dispatcherGateway.submitFailedJob(failedJobId, FAILED_JOB_NAME, t);
            jobIdsFuture.complete(Collections.singletonList(failedJobId));
        } else {
            jobIdsFuture.completeExceptionally(new ApplicationExecutionException("Could not execute application.", t));
        }
    }
}

Example 3 with PipelineExecutorServiceLoader

use of org.apache.flink.core.execution.PipelineExecutorServiceLoader in project flink by apache.

the class DetachedApplicationRunner method tryExecuteJobs.

private List<JobID> tryExecuteJobs(final DispatcherGateway dispatcherGateway, final PackagedProgram program, final Configuration configuration) {
    configuration.set(DeploymentOptions.ATTACHED, false);
    final List<JobID> applicationJobIds = new ArrayList<>();
    final PipelineExecutorServiceLoader executorServiceLoader = new WebSubmissionExecutorServiceLoader(applicationJobIds, dispatcherGateway);
    try {
        ClientUtils.executeProgram(executorServiceLoader, configuration, program, enforceSingleJobExecution, true);
    } catch (ProgramInvocationException e) {
        LOG.warn("Could not execute application: ", e);
        throw new FlinkRuntimeException("Could not execute application.", e);
    }
    return applicationJobIds;
}