Examples with PortablePipelineRunner org.apache.beam.runners.jobsubmission.PortablePipelineRunner used on opensource projects

Example 1 with PortablePipelineRunner

use of org.apache.beam.runners.jobsubmission.PortablePipelineRunner in project beam by apache.

the class FlinkJobInvoker method invokeWithExecutor.

@Override
protected JobInvocation invokeWithExecutor(RunnerApi.Pipeline pipeline, Struct options, @Nullable String retrievalToken, ListeningExecutorService executorService) {
    // TODO: How to make Java/Python agree on names of keys and their values?
    LOG.trace("Parsing pipeline options");
    FlinkPipelineOptions flinkOptions = PipelineOptionsTranslation.fromProto(options).as(FlinkPipelineOptions.class);
    String invocationId = String.format("%s_%s", flinkOptions.getJobName(), UUID.randomUUID().toString());
    if (FlinkPipelineOptions.AUTO.equals(flinkOptions.getFlinkMaster())) {
        flinkOptions.setFlinkMaster(serverConfig.getFlinkMaster());
    }
    PortablePipelineOptions portableOptions = flinkOptions.as(PortablePipelineOptions.class);
    PortablePipelineRunner pipelineRunner;
    if (Strings.isNullOrEmpty(portableOptions.getOutputExecutablePath())) {
        pipelineRunner = new FlinkPipelineRunner(flinkOptions, serverConfig.getFlinkConfDir(), detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader(), flinkOptions));
    } else {
        pipelineRunner = new PortablePipelineJarCreator(FlinkPipelineRunner.class);
    }
    flinkOptions.setRunner(null);
    LOG.info("Invoking job {} with pipeline runner {}", invocationId, pipelineRunner);
    return createJobInvocation(invocationId, retrievalToken, executorService, pipeline, flinkOptions, pipelineRunner);
}

Example 2 with PortablePipelineRunner

use of org.apache.beam.runners.jobsubmission.PortablePipelineRunner in project beam by apache.

the class SamzaJobInvoker method invokeWithExecutor.

@Override
protected JobInvocation invokeWithExecutor(RunnerApi.Pipeline pipeline, Struct options, @Nullable String retrievalToken, ListeningExecutorService executorService) {
    LOG.trace("Parsing pipeline options");
    final SamzaPortablePipelineOptions samzaOptions = PipelineOptionsTranslation.fromProto(options).as(SamzaPortablePipelineOptions.class);
    final PortablePipelineRunner pipelineRunner;
    if (Strings.isNullOrEmpty(samzaOptions.getOutputExecutablePath())) {
        pipelineRunner = new SamzaPipelineRunner(samzaOptions);
    } else {
        /*
       * To support --output_executable_path where bundles the input pipeline along with all
       * artifacts, etc. required to run the pipeline into a jar that can be executed later.
       */
        pipelineRunner = new PortablePipelineJarCreator(SamzaPipelineRunner.class);
    }
    final String invocationId = String.format("%s_%s", samzaOptions.getJobName(), UUID.randomUUID().toString());
    final JobInfo jobInfo = JobInfo.create(invocationId, samzaOptions.getJobName(), retrievalToken, options);
    return new JobInvocation(jobInfo, executorService, pipeline, pipelineRunner);
}

Example 3 with PortablePipelineRunner

use of org.apache.beam.runners.jobsubmission.PortablePipelineRunner in project beam by apache.

the class SparkJobInvoker method createJobInvocation.

static JobInvocation createJobInvocation(String invocationId, String retrievalToken, ListeningExecutorService executorService, Pipeline pipeline, SparkPipelineOptions sparkOptions) {
    JobInfo jobInfo = JobInfo.create(invocationId, sparkOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(sparkOptions));
    PortablePipelineRunner pipelineRunner;
    if (Strings.isNullOrEmpty(sparkOptions.as(PortablePipelineOptions.class).getOutputExecutablePath())) {
        pipelineRunner = new SparkPipelineRunner(sparkOptions);
    } else {
        pipelineRunner = new PortablePipelineJarCreator(SparkPipelineRunner.class);
    }
    return new JobInvocation(jobInfo, executorService, pipeline, pipelineRunner);
}