Examples with ConfigBuilder org.apache.beam.runners.samza.translation.ConfigBuilder used on opensource projects

Example 1 with ConfigBuilder

use of org.apache.beam.runners.samza.translation.ConfigBuilder in project beam by apache.

the class SamzaRunner method run.

@Override
public SamzaPipelineResult run(Pipeline pipeline) {
    // performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    MetricsEnvironment.setMetricsSupported(true);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Pre-processed Beam pipeline in dot format:\n{}", PipelineDotRenderer.toDotString(pipeline));
        LOG.debug("Pre-processed Beam pipeline in json format:\n{}", PipelineJsonRenderer.toJsonString(pipeline));
    }
    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());
    final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
    LOG.info("Beam pipeline DOT graph:\n{}", dotGraph);
    final String jsonGraph = PipelineJsonRenderer.toJsonString(pipeline);
    LOG.info("Beam pipeline JSON graph:\n{}", jsonGraph);
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
    configBuilder.put(BEAM_DOT_GRAPH, dotGraph);
    configBuilder.put(BEAM_JSON_GRAPH, jsonGraph);
    final Config config = configBuilder.build();
    options.setConfigOverride(config);
    if (listener != null) {
        listener.onInit(config, options);
    }
    final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
    final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
    final StreamApplication app = appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory());
        appDescriptor.withMetricsReporterFactories(reporterFactories);
        SamzaPipelineTranslator.translate(pipeline, new TranslationContext(appDescriptor, idMap, options));
    };
    // perform a final round of validation for the pipeline options now that all configs are
    // generated
    SamzaPipelineOptionsValidator.validate(options);
    ApplicationRunner runner = runSamzaApp(app, config);
    return new SamzaPipelineResult(runner, executionContext, listener, config);
}

Example 2 with ConfigBuilder

use of org.apache.beam.runners.samza.translation.ConfigBuilder in project beam by apache.

the class SamzaRunner method runPortablePipeline.

public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline, JobInfo jobInfo) {
    final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
    LOG.info("Portable pipeline to run DOT graph:\n{}", dotGraph);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options);
    configBuilder.put(BEAM_DOT_GRAPH, dotGraph);
    final Config config = configBuilder.build();
    options.setConfigOverride(config);
    if (listener != null) {
        listener.onInit(config, options);
    }
    final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
    final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
    final StreamApplication app = appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory()).withMetricsReporterFactories(reporterFactories);
        SamzaPortablePipelineTranslator.translate(pipeline, new PortableTranslationContext(appDescriptor, options, jobInfo));
    };
    ApplicationRunner runner = runSamzaApp(app, config);
    return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config);
}