Examples with StreamingApplication com.datatorrent.api.StreamingApplication used on opensource projects

Example 1 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project beam by apache.

the class ApexRunner method run.

@Override
public ApexRunnerResult run(final Pipeline pipeline) {
    pipeline.replaceAll(getOverrides());
    final ApexPipelineTranslator translator = new ApexPipelineTranslator(options);
    final AtomicReference<DAG> apexDAG = new AtomicReference<>();
    StreamingApplication apexApp = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            apexDAG.set(dag);
            dag.setAttribute(DAGContext.APPLICATION_NAME, options.getApplicationName());
            translator.translate(pipeline, dag);
        }
    };
    Properties configProperties = new Properties();
    try {
        if (options.getConfigFile() != null) {
            URI configURL = new URI(options.getConfigFile());
            if (CLASSPATH_SCHEME.equals(configURL.getScheme())) {
                InputStream is = this.getClass().getResourceAsStream(configURL.getPath());
                if (is != null) {
                    configProperties.load(is);
                    is.close();
                }
            } else {
                if (!configURL.isAbsolute()) {
                    // resolve as local file name
                    File f = new File(options.getConfigFile());
                    configURL = f.toURI();
                }
                try (InputStream is = configURL.toURL().openStream()) {
                    configProperties.load(is);
                }
            }
        }
    } catch (IOException | URISyntaxException ex) {
        throw new RuntimeException("Error loading properties", ex);
    }
    if (options.isEmbeddedExecution()) {
        EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
        Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
        launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
        if (options.isEmbeddedExecutionDebugMode()) {
            // turns off timeout checking for operator progress
            launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
        }
        Configuration conf = new Configuration(false);
        ApexYarnLauncher.addProperties(conf, configProperties);
        try {
            if (translateOnly) {
                launcher.prepareDAG(apexApp, conf);
                return new ApexRunnerResult(launcher.getDAG(), null);
            }
            ApexRunner.ASSERTION_ERROR.set(null);
            AppHandle apexAppResult = launcher.launchApp(apexApp, conf, launchAttributes);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (Exception e) {
            Throwables.throwIfUnchecked(e);
            throw new RuntimeException(e);
        }
    } else {
        try {
            ApexYarnLauncher yarnLauncher = new ApexYarnLauncher();
            AppHandle apexAppResult = yarnLauncher.launchApp(apexApp, configProperties);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (IOException e) {
            throw new RuntimeException("Failed to launch the application on YARN.", e);
        }
    }
}

Example 2 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project beam by apache.

the class ApexYarnLauncherTest method testProxyLauncher.

@Test
public void testProxyLauncher() throws Exception {
    // use the embedded launcher to build the DAG only
    EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.setAttribute(DAGContext.APPLICATION_NAME, "DummyApp");
        }
    };
    Configuration conf = new Configuration(false);
    DAG dag = embeddedLauncher.prepareDAG(app, conf);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    Properties configProperties = new Properties();
    ApexYarnLauncher launcher = new ApexYarnLauncher();
    launcher.launchApp(new MockApexYarnLauncherParams(dag, launchAttributes, configProperties));
}

Example 3 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-core by apache.

the class LogicalPlanConfigurationTest method testModuleUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testModuleUnifierLevelAttributes() {
    class DummyOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
                output.emit(tuple);
            }
        };

        public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<>();
    }
    class DummyOutputOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
            }
        };
    }
    class TestUnifierAttributeModule implements Module {

        public transient ProxyInputPort<Integer> moduleInput = new ProxyInputPort<>();

        public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<>();

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
            dag.setOperatorAttribute(dummyOperator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<DummyOperator>(3));
            dag.setUnifierAttribute(dummyOperator.output, OperatorContext.TIMEOUT_WINDOW_COUNT, 2);
            moduleInput.set(dummyOperator.input);
            moduleOutput.set(dummyOperator.output);
        }
    }
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            Module m1 = dag.addModule("TestModule", new TestUnifierAttributeModule());
            DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
            dag.addStream("Module To Operator", ((TestUnifierAttributeModule) m1).moduleOutput, dummyOutputOperator.input);
        }
    };
    String appName = "UnifierApp";
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new MockStorageAgent());
    dagBuilder.prepareDAG(dag, app, appName);
    LogicalPlan.OperatorMeta ometa = dag.getOperatorMeta("TestModule$DummyOperator");
    LogicalPlan.OperatorMeta om = null;
    for (Map.Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ometa.getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("output")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    /*
     * Verify the attribute value after preparing DAG.
     */
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(Context.OperatorContext.TIMEOUT_WINDOW_COUNT));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    LogicalPlan.OperatorMeta operatorMeta = null;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (operator.isUnifier()) {
                operatorMeta = operator.getOperatorMeta();
            }
        }
    }
    /*
     * Verify attribute after physical plan creation with partitioned operators.
     */
    Assert.assertEquals("", Integer.valueOf(2), operatorMeta.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT));
}

Example 4 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-core by apache.

the class LogicalPlanConfigurationTest method testUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testUnifierLevelAttributes() {
    String appName = "app1";
    final GenericTestOperator operator1 = new GenericTestOperator();
    final GenericTestOperator operator2 = new GenericTestOperator();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", operator1);
            dag.addOperator("operator2", operator2);
            dag.addStream("s1", operator1.outport1, operator2.inport1);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.outputport.outport1.unifier." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.outputport.outport1.unifier." + OperatorContext.MEMORY_MB.getName(), "512");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    OperatorMeta om = null;
    for (Map.Entry<OutputPortMeta, StreamMeta> entry : dag.getOperatorMeta("operator1").getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("outport1")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", Integer.valueOf(512), om.getValue(OperatorContext.MEMORY_MB));
}

Example 5 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-core by apache.

the class LogicalPlanConfigurationTest method testOperatorLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testOperatorLevelAttributes() {
    String appName = "app1";
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", GenericTestOperator.class);
            dag.addOperator("operator2", GenericTestOperator.class);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
    props.put(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.STATS_LISTENERS.getName(), PartitionLoadWatch.class.getName());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "20");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    Assert.assertEquals("", Integer.valueOf(20), dag.getOperatorMeta("operator1").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", Integer.valueOf(2), dag.getOperatorMeta("operator2").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", PartitionLoadWatch.class, dag.getOperatorMeta("operator2").getValue(OperatorContext.STATS_LISTENERS).toArray()[0].getClass());
}