Examples with ApexPipelineTranslator org.apache.beam.runners.apex.translation.ApexPipelineTranslator used on opensource projects

Example 1 with ApexPipelineTranslator

use of org.apache.beam.runners.apex.translation.ApexPipelineTranslator in project beam by apache.

the class ApexRunner method run.

@Override
public ApexRunnerResult run(final Pipeline pipeline) {
    pipeline.replaceAll(getOverrides());
    final ApexPipelineTranslator translator = new ApexPipelineTranslator(options);
    final AtomicReference<DAG> apexDAG = new AtomicReference<>();
    StreamingApplication apexApp = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            apexDAG.set(dag);
            dag.setAttribute(DAGContext.APPLICATION_NAME, options.getApplicationName());
            translator.translate(pipeline, dag);
        }
    };
    Properties configProperties = new Properties();
    try {
        if (options.getConfigFile() != null) {
            URI configURL = new URI(options.getConfigFile());
            if (CLASSPATH_SCHEME.equals(configURL.getScheme())) {
                InputStream is = this.getClass().getResourceAsStream(configURL.getPath());
                if (is != null) {
                    configProperties.load(is);
                    is.close();
                }
            } else {
                if (!configURL.isAbsolute()) {
                    // resolve as local file name
                    File f = new File(options.getConfigFile());
                    configURL = f.toURI();
                }
                try (InputStream is = configURL.toURL().openStream()) {
                    configProperties.load(is);
                }
            }
        }
    } catch (IOException | URISyntaxException ex) {
        throw new RuntimeException("Error loading properties", ex);
    }
    if (options.isEmbeddedExecution()) {
        EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
        Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
        launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
        if (options.isEmbeddedExecutionDebugMode()) {
            // turns off timeout checking for operator progress
            launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
        }
        Configuration conf = new Configuration(false);
        ApexYarnLauncher.addProperties(conf, configProperties);
        try {
            if (translateOnly) {
                launcher.prepareDAG(apexApp, conf);
                return new ApexRunnerResult(launcher.getDAG(), null);
            }
            ApexRunner.ASSERTION_ERROR.set(null);
            AppHandle apexAppResult = launcher.launchApp(apexApp, conf, launchAttributes);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (Exception e) {
            Throwables.throwIfUnchecked(e);
            throw new RuntimeException(e);
        }
    } else {
        try {
            ApexYarnLauncher yarnLauncher = new ApexYarnLauncher();
            AppHandle apexAppResult = yarnLauncher.launchApp(apexApp, configProperties);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (IOException e) {
            throw new RuntimeException("Failed to launch the application on YARN.", e);
        }
    }
}