Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 16 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class AccumuloInputOperatorTest method testScan.

@Test
public void testScan() {
    try {
        AccumuloTestHelper.getConnector();
        AccumuloTestHelper.deleteTable();
        AccumuloTestHelper.createTable();
        AccumuloTestHelper.populateAccumulo();
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        dag.setAttribute(DAG.APPLICATION_NAME, "AccumuloInputTest");
        TestAccumuloInputOperator taip = dag.addOperator("testaccumuloinput", TestAccumuloInputOperator.class);
        AccumuloTupleCollector tc = dag.addOperator("tuplecollector", AccumuloTupleCollector.class);
        dag.addStream("ss", taip.outputPort, tc.inputPort);
        taip.getStore().setTableName("tab1");
        taip.getStore().setZookeeperHost("127.0.0.1");
        taip.getStore().setInstanceName("instance");
        taip.getStore().setUserName("root");
        taip.getStore().setPassword("pass");
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        lc.run(30000);
        List<AccumuloTuple> tuples = AccumuloTupleCollector.tuples;
        Assert.assertTrue(tuples.size() > 0);
        AccumuloTuple tuple = AccumuloTestHelper.findTuple(tuples, "row0", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-0");
        tuple = AccumuloTestHelper.findTuple(tuples, "row499", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row499");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-499-0");
    } catch (Exception ex) {
        logger.error(ex.getMessage());
        assert false;
    }
}

Example 17 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class TwitterDumpApplicationTest method testPopulateDAG.

@Test
public void testPopulateDAG() throws Exception {
    Configuration configuration = new Configuration(false);
    LocalMode lm = LocalMode.newInstance();
    DAG prepareDAG = lm.prepareDAG(new TwitterDumpApplication(), configuration);
    DAG clonedDAG = lm.cloneDAG();
    assertEquals("Serialization", prepareDAG, clonedDAG);
}

Example 18 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class XmlParserApplicationTest method testApplication.

@Test
public void testApplication() {
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        XmlDataEmitterOperator input = dag.addOperator("data", new XmlDataEmitterOperator());
        XmlParser parser = dag.addOperator("xmlparser", new XmlParser());
        ResultCollector rc = dag.addOperator("rc", new ResultCollector());
        dag.getMeta(parser).getMeta(parser.out).getAttributes().put(Context.PortContext.TUPLE_CLASS, org.apache.apex.malhar.lib.parser.XmlParserTest.EmployeeBean.class);
        ConsoleOutputOperator xmlObjectOp = dag.addOperator("xmlObjectOp", new ConsoleOutputOperator());
        xmlObjectOp.setDebug(true);
        dag.addStream("input", input.output, parser.in);
        dag.addStream("output", parser.parsedOutput, xmlObjectOp.input);
        dag.addStream("pojo", parser.out, rc.input);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return TupleCount == 1;
            }
        });
        // runs for 10 seconds and quits
        lc.run(10000);
        Assert.assertEquals(1, TupleCount);
        Assert.assertEquals("john", obj.getName());
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

Example 19 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class AbstractWindowedOperatorBenchmarkApp method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    TestStatsListener sl = new TestStatsListener();
    sl.adjustRate = conf.getBoolean("dt.ManagedStateBenchmark.adjustRate", false);
    G generator = createGenerator();
    dag.addOperator("Generator", generator);
    // generator.setRange(timeRange);
    dag.setAttribute(generator, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) sl));
    O windowedOperator = createWindowedOperator(conf);
    dag.addOperator("windowedOperator", windowedOperator);
    dag.setAttribute(windowedOperator, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) sl));
    // dag.addStream("Data", generator.data, windowedOperator.input).setLocality(Locality.CONTAINER_LOCAL);
    connectGeneratorToWindowedOperator(dag, generator, windowedOperator);
    // WatermarkGenerator watermarkGenerator = new WatermarkGenerator();
    // dag.addOperator("WatermarkGenerator", watermarkGenerator);
    // dag.addStream("Control", watermarkGenerator.control, windowedOperator.controlInput)
    // .setLocality(Locality.CONTAINER_LOCAL);
    DevNull output = dag.addOperator("output", new DevNull());
    dag.addStream("output", windowedOperator.output, output.data).setLocality(Locality.CONTAINER_LOCAL);
}

Example 20 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class ApplicationFixedTest method testApplication.

@Test
public void testApplication() throws IOException, Exception {
    LocalMode lma = LocalMode.newInstance();
    new ApplicationFixed().populateDAG(lma.getDAG(), new Configuration(false));
    DAG dag = lma.cloneDAG();
    FixedTuplesInputOperator wordGenerator = (FixedTuplesInputOperator) dag.getOperatorMeta("WordGenerator").getOperator();
    Assert.assertEquals("Queue Capacity", ApplicationFixed.QUEUE_CAPACITY, (int) dag.getMeta(wordGenerator).getMeta(wordGenerator.output).getValue(PortContext.QUEUE_CAPACITY));
    LocalMode.Controller lc = lma.getController();
    lc.run(60000);
}