Examples with JobInfo org.apache.beam.runners.fnexecution.provisioning.JobInfo used on opensource projects

Example 6 with JobInfo

use of org.apache.beam.runners.fnexecution.provisioning.JobInfo in project beam by apache.

the class ReferenceCountingExecutableStageContextFactoryTest method testCatchThrowablesAndLogThem.

@Test
public void testCatchThrowablesAndLogThem() throws Exception {
    PrintStream oldErr = System.err;
    oldErr.flush();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream newErr = new PrintStream(baos);
    try {
        System.setErr(newErr);
        Creator creator = mock(Creator.class);
        ExecutableStageContext c1 = mock(ExecutableStageContext.class);
        when(creator.apply(any(JobInfo.class))).thenReturn(c1);
        // throw an Throwable and ensure that it is caught and logged.
        doThrow(new NoClassDefFoundError()).when(c1).close();
        ReferenceCountingExecutableStageContextFactory factory = ReferenceCountingExecutableStageContextFactory.create(creator, (x) -> true);
        JobInfo jobA = mock(JobInfo.class);
        when(jobA.jobId()).thenReturn("jobA");
        ExecutableStageContext ac1A = factory.get(jobA);
        factory.release(ac1A);
        newErr.flush();
        String output = new String(baos.toByteArray(), Charsets.UTF_8);
        // Ensure that the error is logged
        assertTrue(output.contains("Unable to close ExecutableStageContext"));
    } finally {
        newErr.flush();
        System.setErr(oldErr);
    }
}

Example 7 with JobInfo

use of org.apache.beam.runners.fnexecution.provisioning.JobInfo in project beam by apache.

the class JobInvocationTest method setup.

@Before
public void setup() {
    executorService = Executors.newFixedThreadPool(1);
    JobInfo jobInfo = JobInfo.create("jobid", "jobName", "retrievalToken", Struct.getDefaultInstance());
    ListeningExecutorService listeningExecutorService = MoreExecutors.listeningDecorator(executorService);
    Pipeline pipeline = Pipeline.create();
    runner = new ControllablePipelineRunner();
    jobInvocation = new JobInvocation(jobInfo, listeningExecutorService, PipelineTranslation.toProto(pipeline), runner);
}

Example 8 with JobInfo

use of org.apache.beam.runners.fnexecution.provisioning.JobInfo in project beam by apache.

the class SamzaPipelineRunner method run.

@Override
public PortablePipelineResult run(final RunnerApi.Pipeline pipeline, JobInfo jobInfo) {
    // Expand any splittable DoFns within the graph to enable sizing and splitting of bundles.
    RunnerApi.Pipeline pipelineWithSdfExpanded = ProtoOverrides.updateTransform(PTransformTranslation.PAR_DO_TRANSFORM_URN, pipeline, SplittableParDoExpander.createSizedReplacement());
    // Don't let the fuser fuse any subcomponents of native transforms.
    RunnerApi.Pipeline trimmedPipeline = TrivialNativeTransformExpander.forKnownUrns(pipelineWithSdfExpanded, SamzaPortablePipelineTranslator.knownUrns());
    // Fused pipeline proto.
    // TODO: Consider supporting partially-fused graphs.
    RunnerApi.Pipeline fusedPipeline = trimmedPipeline.getComponents().getTransformsMap().values().stream().anyMatch(proto -> ExecutableStage.URN.equals(proto.getSpec().getUrn())) ? trimmedPipeline : GreedyPipelineFuser.fuse(trimmedPipeline).toPipeline();
    LOG.info("Portable pipeline to run:");
    LOG.info(PipelineDotRenderer.toDotString(fusedPipeline));
    // the pipeline option coming from sdk will set the sdk specific runner which will break
    // serialization
    // so we need to reset the runner here to a valid Java runner
    options.setRunner(SamzaRunner.class);
    try {
        final SamzaRunner runner = SamzaRunner.fromOptions(options);
        final PortablePipelineResult result = runner.runPortablePipeline(fusedPipeline, jobInfo);
        final SamzaExecutionEnvironment exeEnv = options.getSamzaExecutionEnvironment();
        if (exeEnv == SamzaExecutionEnvironment.LOCAL || exeEnv == SamzaExecutionEnvironment.STANDALONE) {
            // Make run() sync for local mode
            result.waitUntilFinish();
        }
        return result;
    } catch (Exception e) {
        throw new RuntimeException("Failed to invoke samza job", e);
    }
}

Example 9 with JobInfo

use of org.apache.beam.runners.fnexecution.provisioning.JobInfo in project beam by apache.

the class SamzaRunner method runPortablePipeline.

public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline, JobInfo jobInfo) {
    final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
    LOG.info("Portable pipeline to run DOT graph:\n{}", dotGraph);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options);
    configBuilder.put(BEAM_DOT_GRAPH, dotGraph);
    final Config config = configBuilder.build();
    options.setConfigOverride(config);
    if (listener != null) {
        listener.onInit(config, options);
    }
    final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
    final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
    final StreamApplication app = appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory()).withMetricsReporterFactories(reporterFactories);
        SamzaPortablePipelineTranslator.translate(pipeline, new PortableTranslationContext(appDescriptor, options, jobInfo));
    };
    ApplicationRunner runner = runSamzaApp(app, config);
    return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config);
}

Example 10 with JobInfo

use of org.apache.beam.runners.fnexecution.provisioning.JobInfo in project beam by apache.

the class SparkPipelineRunner method main.

/**
 * Main method to be called only as the entry point to an executable jar with structure as defined
 * in {@link PortablePipelineJarUtils}.
 */
public static void main(String[] args) throws Exception {
    // Register standard file systems.
    FileSystems.setDefaultPipelineOptions(PipelineOptionsFactory.create());
    SparkPipelineRunnerConfiguration configuration = parseArgs(args);
    String baseJobName = configuration.baseJobName == null ? PortablePipelineJarUtils.getDefaultJobName() : configuration.baseJobName;
    Preconditions.checkArgument(baseJobName != null, "No default job name found. Job name must be set using --base-job-name.");
    Pipeline pipeline = PortablePipelineJarUtils.getPipelineFromClasspath(baseJobName);
    Struct originalOptions = PortablePipelineJarUtils.getPipelineOptionsFromClasspath(baseJobName);
    // The retrieval token is only required by the legacy artifact service, which the Spark runner
    // no longer uses.
    String retrievalToken = ArtifactApi.CommitManifestResponse.Constants.NO_ARTIFACTS_STAGED_TOKEN.getValueDescriptor().getOptions().getExtension(RunnerApi.beamConstant);
    SparkPipelineOptions sparkOptions = PipelineOptionsTranslation.fromProto(originalOptions).as(SparkPipelineOptions.class);
    String invocationId = String.format("%s_%s", sparkOptions.getJobName(), UUID.randomUUID().toString());
    if (sparkOptions.getAppName() == null) {
        LOG.debug("App name was null. Using invocationId {}", invocationId);
        sparkOptions.setAppName(invocationId);
    }
    SparkPipelineRunner runner = new SparkPipelineRunner(sparkOptions);
    JobInfo jobInfo = JobInfo.create(invocationId, sparkOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(sparkOptions));
    try {
        runner.run(pipeline, jobInfo);
    } catch (Exception e) {
        throw new RuntimeException(String.format("Job %s failed.", invocationId), e);
    }
    LOG.info("Job {} finished successfully.", invocationId);
}