Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 61 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class ApexStreamImpl method runEmbedded.

@Override
public void runEmbedded(boolean async, long duration, Callable<Boolean> exitCondition) {
    LocalMode lma = LocalMode.newInstance();
    populateDag(lma.getDAG());
    DAG dag = lma.getDAG();
    LocalMode.Controller lc = lma.getController();
    if (lc instanceof StramLocalCluster) {
        ((StramLocalCluster) lc).setExitCondition(exitCondition);
    }
    if (async) {
        lc.runAsync();
    } else {
        if (duration >= 0) {
            lc.run(duration);
        } else {
            lc.run();
        }
    }
}

Example 62 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class DagMeta method buildDAG.

public DAG buildDAG() {
    DAG dag = new LogicalPlan();
    buildDAG(dag);
    return dag;
}

Example 63 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class SmtpOutputOperatorTest method testProperties.

@Test
public void testProperties() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.subject", subject);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.content", content);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.from", from);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.smtpHost", "127.0.0.1");
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.smtpUserName", from);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.smtpPassword", "<password>");
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.recipients.TO", to + "," + cc);
    conf.set(StreamingApplication.DT_PREFIX + "operator.o1.prop.recipients.CC", cc);
    final AtomicReference<SmtpOutputOperator> o1 = new AtomicReference<SmtpOutputOperator>();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            o1.set(dag.addOperator("o1", new SmtpOutputOperator()));
        }
    };
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    Assert.assertEquals("checking TO list", to + "," + cc, o1.get().getRecipients().get("TO"));
    Assert.assertEquals("checking CC list", cc, o1.get().getRecipients().get("CC"));
}

Example 64 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class SliceBloomFilterTest method testBloomFilterForApplication.

/**
 * Just test SliceBloomFilter can be used by operator. such as it is serializable etc
 * @throws Exception
 */
@Test
public void testBloomFilterForApplication() throws Exception {
    Configuration conf = new Configuration(false);
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    TestInputOperator generator = new TestInputOperator();
    dag.addOperator("Generator", generator);
    FilterOperator filterOperator = new FilterOperator();
    dag.addOperator("filterOperator", filterOperator);
    dag.addStream("Data", generator.data, filterOperator.input).setLocality(Locality.CONTAINER_LOCAL);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.run(3000);
    lc.shutdown();
}

Example 65 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class RedisPOJOOperatorTest method testInputOperator.

@Test
public void testInputOperator() throws IOException {
    @SuppressWarnings("unused") Class<?> clazz = org.codehaus.janino.CompilerFactory.class;
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(100);
    Map<String, String> value = new HashMap<String, String>();
    value.put("Column1", "abc");
    value.put("Column2", "1");
    Map<String, String> value1 = new HashMap<String, String>();
    value1.put("Column1", "def");
    value1.put("Column2", "2");
    Map<String, String> value2 = new HashMap<String, String>();
    value2.put("Column1", "ghi");
    value2.put("Column2", "3");
    testStore.put("test_abc_in", value);
    testStore.put("test_def_in", value1);
    testStore.put("test_ghi_in", value2);
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        RedisPOJOInputOperator inputOperator = dag.addOperator("input", new RedisPOJOInputOperator());
        final ObjectCollectorModule collector = dag.addOperator("collector", new ObjectCollectorModule());
        ArrayList<FieldInfo> fields = new ArrayList<FieldInfo>();
        fields.add(new FieldInfo("Column1", "stringValue", SupportType.STRING));
        fields.add(new FieldInfo("Column2", "intValue", SupportType.INTEGER));
        inputOperator.setDataColumns(fields);
        inputOperator.setOutputClass(TestClass.class.getName());
        inputOperator.setStore(operatorStore);
        dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
        final LocalMode.Controller lc = lma.getController();
        new Thread("LocalClusterController") {

            @Override
            public void run() {
                long startTms = System.currentTimeMillis();
                long timeout = 10000L;
                try {
                    Thread.sleep(1000);
                    while (System.currentTimeMillis() - startTms < timeout) {
                        if (ObjectCollectorModule.resultMap.size() < 3) {
                            Thread.sleep(10);
                        } else {
                            break;
                        }
                    }
                } catch (InterruptedException ex) {
                // 
                }
                lc.shutdown();
            }
        }.start();
        lc.run();
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_abc_in"));
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_def_in"));
        Assert.assertTrue(ObjectCollectorModule.resultMap.containsKey("test_ghi_in"));
        TestClass a = (TestClass) ObjectCollectorModule.resultMap.get("test_abc_in");
        Assert.assertNotNull(a);
        Assert.assertEquals("abc", a.stringValue);
        Assert.assertEquals("1", a.intValue.toString());
    } finally {
        for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
            testStore.remove(entry.getKey());
        }
        testStore.disconnect();
    }
}