Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 11 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class ApexRunnerTest method testParDoChaining.

@Test
public void testParDoChaining() throws Exception {
    Pipeline p = Pipeline.create();
    long numElements = 1000;
    PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements));
    PAssert.thatSingleton(input.apply("Count", Count.<Long>globally())).isEqualTo(numElements);
    ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
    DAG dag = TestApexRunner.translate(p, options);
    String[] expectedThreadLocal = { "/CreateActual/FilterActuals/Window.Assign" };
    Set<String> actualThreadLocal = Sets.newHashSet();
    for (DAG.StreamMeta sm : dag.getAllStreamsMeta()) {
        DAG.OutputPortMeta opm = sm.getSource();
        if (sm.getLocality() == Locality.THREAD_LOCAL) {
            String name = opm.getOperatorMeta().getName();
            String prefix = "PAssert$";
            if (name.startsWith(prefix)) {
                // remove indeterministic prefix
                name = name.substring(prefix.length() + 1);
            }
            actualThreadLocal.add(name);
        }
    }
    Assert.assertThat(actualThreadLocal, Matchers.hasItems(expectedThreadLocal));
}

Example 12 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class ApexYarnLauncherTest method testProxyLauncher.

@Test
public void testProxyLauncher() throws Exception {
    // use the embedded launcher to build the DAG only
    EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.setAttribute(DAGContext.APPLICATION_NAME, "DummyApp");
        }
    };
    Configuration conf = new Configuration(false);
    DAG dag = embeddedLauncher.prepareDAG(app, conf);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    Properties configProperties = new Properties();
    ApexYarnLauncher launcher = new ApexYarnLauncher();
    launcher.launchApp(new MockApexYarnLauncherParams(dag, launchAttributes, configProperties));
}

Example 13 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class ApexRunnerTest method testConfigProperties.

@Test
public void testConfigProperties() throws Exception {
    String operName = "testProperties";
    ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class);
    // default configuration from class path
    Pipeline p = Pipeline.create();
    Create.Values<Void> empty = Create.empty(VoidCoder.of());
    p.apply(operName, empty);
    DAG dag = TestApexRunner.translate(p, options);
    OperatorMeta t1Meta = dag.getOperatorMeta(operName);
    Assert.assertNotNull(t1Meta);
    Assert.assertEquals(new Integer(32), t1Meta.getValue(OperatorContext.MEMORY_MB));
    File tmp = File.createTempFile("beam-runners-apex-", ".properties");
    tmp.deleteOnExit();
    Properties props = new Properties();
    props.setProperty("apex.operator." + operName + ".attr.MEMORY_MB", "64");
    try (FileOutputStream fos = new FileOutputStream(tmp)) {
        props.store(fos, "");
    }
    options.setConfigFile(tmp.getAbsolutePath());
    dag = TestApexRunner.translate(p, options);
    tmp.delete();
    t1Meta = dag.getOperatorMeta(operName);
    Assert.assertNotNull(t1Meta);
    Assert.assertEquals(new Integer(64), t1Meta.getValue(OperatorContext.MEMORY_MB));
}

Example 14 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class RabbitMQOutputOperatorTest method runTest.

protected void runTest(int testNum) throws IOException {
    RabbitMQMessageReceiver receiver = new RabbitMQMessageReceiver();
    receiver.setup();
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    SourceModule source = dag.addOperator("source", new SourceModule());
    source.setTestNum(testNum);
    RabbitMQOutputOperator collector = dag.addOperator("generator", new RabbitMQOutputOperator());
    collector.setWindowDataManager(new FSWindowDataManager());
    collector.setExchange("testEx");
    dag.addStream("Stream", source.outPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    try {
        Thread.sleep(1000);
        long timeout = 10000L;
        long startTms = System.currentTimeMillis();
        while ((receiver.count < testNum * 3) && (System.currentTimeMillis() - startTms < timeout)) {
            Thread.sleep(100);
        }
    } catch (InterruptedException ex) {
        Assert.fail(ex.getMessage());
    } finally {
        lc.shutdown();
    }
    Assert.assertEquals("emitted value for testNum was ", testNum * 3, receiver.count);
    for (Map.Entry<String, Integer> e : receiver.dataMap.entrySet()) {
        if (e.getKey().equals("a")) {
            Assert.assertEquals("emitted value for 'a' was ", new Integer(2), e.getValue());
        } else if (e.getKey().equals("b")) {
            Assert.assertEquals("emitted value for 'b' was ", new Integer(20), e.getValue());
        } else if (e.getKey().equals("c")) {
            Assert.assertEquals("emitted value for 'c' was ", new Integer(1000), e.getValue());
        }
    }
}

Example 15 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class RedisInputOperatorTest method testIntputOperator.

@Test
public void testIntputOperator() throws IOException {
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(1);
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        RedisKeyValueInputOperator inputOperator = dag.addOperator("input", new RedisKeyValueInputOperator());
        final CollectorModule collector = dag.addOperator("collector", new CollectorModule());
        inputOperator.setStore(operatorStore);
        dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
        final LocalMode.Controller lc = lma.getController();
        new Thread("LocalClusterController") {

            @Override
            public void run() {
                long startTms = System.currentTimeMillis();
                long timeout = 50000L;
                try {
                    Thread.sleep(1000);
                    while (System.currentTimeMillis() - startTms < timeout) {
                        if (CollectorModule.resultMap.size() < 3) {
                            Thread.sleep(10);
                        } else {
                            break;
                        }
                    }
                } catch (InterruptedException ex) {
                // 
                }
                lc.shutdown();
            }
        }.start();
        lc.run();
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_abc", "789")));
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_def", "456")));
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_ghi", "123")));
    } finally {
        for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
            testStore.remove(entry.getKey());
        }
        testStore.disconnect();
    }
}