Search in sources :

Example 1 with ConfigBuilder

use of org.apache.beam.runners.samza.translation.ConfigBuilder in project beam by apache.

the class SamzaRunner method run.

@Override
public SamzaPipelineResult run(Pipeline pipeline) {
    // performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    MetricsEnvironment.setMetricsSupported(true);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Pre-processed Beam pipeline in dot format:\n{}", PipelineDotRenderer.toDotString(pipeline));
        LOG.debug("Pre-processed Beam pipeline in json format:\n{}", PipelineJsonRenderer.toJsonString(pipeline));
    }
    pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());
    final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
    LOG.info("Beam pipeline DOT graph:\n{}", dotGraph);
    final String jsonGraph = PipelineJsonRenderer.toJsonString(pipeline);
    LOG.info("Beam pipeline JSON graph:\n{}", jsonGraph);
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
    configBuilder.put(BEAM_DOT_GRAPH, dotGraph);
    configBuilder.put(BEAM_JSON_GRAPH, jsonGraph);
    final Config config = configBuilder.build();
    options.setConfigOverride(config);
    if (listener != null) {
        listener.onInit(config, options);
    }
    final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
    final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
    final StreamApplication app = appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory());
        appDescriptor.withMetricsReporterFactories(reporterFactories);
        SamzaPipelineTranslator.translate(pipeline, new TranslationContext(appDescriptor, idMap, options));
    };
    // perform a final round of validation for the pipeline options now that all configs are
    // generated
    SamzaPipelineOptionsValidator.validate(options);
    ApplicationRunner runner = runSamzaApp(app, config);
    return new SamzaPipelineResult(runner, executionContext, listener, config);
}
Also used : PViewToIdMapper(org.apache.beam.runners.samza.translation.PViewToIdMapper) PortableTranslationContext(org.apache.beam.runners.samza.translation.PortableTranslationContext) ExperimentalOptions(org.apache.beam.sdk.options.ExperimentalOptions) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) PipelineJsonRenderer(org.apache.beam.runners.samza.util.PipelineJsonRenderer) PipelineRunner(org.apache.beam.sdk.PipelineRunner) Map(java.util.Map) SamzaPipelineTranslator(org.apache.beam.runners.samza.translation.SamzaPipelineTranslator) MetricsReporter(org.apache.samza.metrics.MetricsReporter) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) JobInfo(org.apache.beam.runners.fnexecution.provisioning.JobInfo) PortablePipelineResult(org.apache.beam.runners.jobsubmission.PortablePipelineResult) Pipeline(org.apache.beam.sdk.Pipeline) Iterators(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators) ApplicationRunners(org.apache.samza.runtime.ApplicationRunners) ExternalContext(org.apache.samza.context.ExternalContext) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) PipelineDotRenderer(org.apache.beam.runners.core.construction.renderer.PipelineDotRenderer) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) TranslationContext(org.apache.beam.runners.samza.translation.TranslationContext) ServiceLoader(java.util.ServiceLoader) SplittableParDo(org.apache.beam.runners.core.construction.SplittableParDo) SamzaPortablePipelineTranslator(org.apache.beam.runners.samza.translation.SamzaPortablePipelineTranslator) PipelineOptionsValidator(org.apache.beam.sdk.options.PipelineOptionsValidator) MetricsEnvironment(org.apache.beam.sdk.metrics.MetricsEnvironment) PValue(org.apache.beam.sdk.values.PValue) ConfigBuilder(org.apache.beam.runners.samza.translation.ConfigBuilder) Config(org.apache.samza.config.Config) SamzaTransformOverrides(org.apache.beam.runners.samza.translation.SamzaTransformOverrides) StreamApplication(org.apache.samza.application.StreamApplication) Collections(java.util.Collections) Config(org.apache.samza.config.Config) StreamApplication(org.apache.samza.application.StreamApplication) LoggerFactory(org.slf4j.LoggerFactory) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) PValue(org.apache.beam.sdk.values.PValue) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) ConfigBuilder(org.apache.beam.runners.samza.translation.ConfigBuilder) PortableTranslationContext(org.apache.beam.runners.samza.translation.PortableTranslationContext) TranslationContext(org.apache.beam.runners.samza.translation.TranslationContext)

Example 2 with ConfigBuilder

use of org.apache.beam.runners.samza.translation.ConfigBuilder in project beam by apache.

the class SamzaRunner method runPortablePipeline.

public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline, JobInfo jobInfo) {
    final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
    LOG.info("Portable pipeline to run DOT graph:\n{}", dotGraph);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options);
    configBuilder.put(BEAM_DOT_GRAPH, dotGraph);
    final Config config = configBuilder.build();
    options.setConfigOverride(config);
    if (listener != null) {
        listener.onInit(config, options);
    }
    final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
    final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
    final StreamApplication app = appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory()).withMetricsReporterFactories(reporterFactories);
        SamzaPortablePipelineTranslator.translate(pipeline, new PortableTranslationContext(appDescriptor, options, jobInfo));
    };
    ApplicationRunner runner = runSamzaApp(app, config);
    return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config);
}
Also used : PViewToIdMapper(org.apache.beam.runners.samza.translation.PViewToIdMapper) PortableTranslationContext(org.apache.beam.runners.samza.translation.PortableTranslationContext) ExperimentalOptions(org.apache.beam.sdk.options.ExperimentalOptions) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) PipelineJsonRenderer(org.apache.beam.runners.samza.util.PipelineJsonRenderer) PipelineRunner(org.apache.beam.sdk.PipelineRunner) Map(java.util.Map) SamzaPipelineTranslator(org.apache.beam.runners.samza.translation.SamzaPipelineTranslator) MetricsReporter(org.apache.samza.metrics.MetricsReporter) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) JobInfo(org.apache.beam.runners.fnexecution.provisioning.JobInfo) PortablePipelineResult(org.apache.beam.runners.jobsubmission.PortablePipelineResult) Pipeline(org.apache.beam.sdk.Pipeline) Iterators(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators) ApplicationRunners(org.apache.samza.runtime.ApplicationRunners) ExternalContext(org.apache.samza.context.ExternalContext) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) PipelineDotRenderer(org.apache.beam.runners.core.construction.renderer.PipelineDotRenderer) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) TranslationContext(org.apache.beam.runners.samza.translation.TranslationContext) ServiceLoader(java.util.ServiceLoader) SplittableParDo(org.apache.beam.runners.core.construction.SplittableParDo) SamzaPortablePipelineTranslator(org.apache.beam.runners.samza.translation.SamzaPortablePipelineTranslator) PipelineOptionsValidator(org.apache.beam.sdk.options.PipelineOptionsValidator) MetricsEnvironment(org.apache.beam.sdk.metrics.MetricsEnvironment) PValue(org.apache.beam.sdk.values.PValue) ConfigBuilder(org.apache.beam.runners.samza.translation.ConfigBuilder) Config(org.apache.samza.config.Config) SamzaTransformOverrides(org.apache.beam.runners.samza.translation.SamzaTransformOverrides) StreamApplication(org.apache.samza.application.StreamApplication) Collections(java.util.Collections) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) Config(org.apache.samza.config.Config) StreamApplication(org.apache.samza.application.StreamApplication) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) ConfigBuilder(org.apache.beam.runners.samza.translation.ConfigBuilder) LoggerFactory(org.slf4j.LoggerFactory) MetricsReporterFactory(org.apache.samza.metrics.MetricsReporterFactory) PortableTranslationContext(org.apache.beam.runners.samza.translation.PortableTranslationContext)

Aggregations

Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 Iterator (java.util.Iterator)2 Map (java.util.Map)2 ServiceLoader (java.util.ServiceLoader)2 RunnerApi (org.apache.beam.model.pipeline.v1.RunnerApi)2 SplittableParDo (org.apache.beam.runners.core.construction.SplittableParDo)2 PipelineDotRenderer (org.apache.beam.runners.core.construction.renderer.PipelineDotRenderer)2 JobInfo (org.apache.beam.runners.fnexecution.provisioning.JobInfo)2 PortablePipelineResult (org.apache.beam.runners.jobsubmission.PortablePipelineResult)2 ConfigBuilder (org.apache.beam.runners.samza.translation.ConfigBuilder)2 PViewToIdMapper (org.apache.beam.runners.samza.translation.PViewToIdMapper)2 PortableTranslationContext (org.apache.beam.runners.samza.translation.PortableTranslationContext)2 SamzaPipelineTranslator (org.apache.beam.runners.samza.translation.SamzaPipelineTranslator)2 SamzaPortablePipelineTranslator (org.apache.beam.runners.samza.translation.SamzaPortablePipelineTranslator)2 SamzaTransformOverrides (org.apache.beam.runners.samza.translation.SamzaTransformOverrides)2 TranslationContext (org.apache.beam.runners.samza.translation.TranslationContext)2 PipelineJsonRenderer (org.apache.beam.runners.samza.util.PipelineJsonRenderer)2 Pipeline (org.apache.beam.sdk.Pipeline)2 PipelineRunner (org.apache.beam.sdk.PipelineRunner)2