Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 46 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KeyValueStoreOperatorTest method testInputOperator.

public void testInputOperator() throws Exception {
    testStore.connect();
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        @SuppressWarnings("unchecked") InputOperator<S> inputOperator = dag.addOperator("input", new InputOperator<S>());
        CollectorModule<Object> collector = dag.addOperator("collector", new CollectorModule<Object>());
        inputOperator.addKey("test_abc");
        inputOperator.addKey("test_def");
        inputOperator.addKey("test_ghi");
        inputOperator.setStore(operatorStore);
        dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
        final LocalMode.Controller lc = lma.getController();
        lc.run(3000);
        lc.shutdown();
        Assert.assertEquals("789", CollectorModule.resultMap.get("test_abc"));
        Assert.assertEquals("456", CollectorModule.resultMap.get("test_def"));
        Assert.assertEquals("123", CollectorModule.resultMap.get("test_ghi"));
    } finally {
        testStore.remove("test_abc");
        testStore.remove("test_def");
        testStore.remove("test_ghi");
        testStore.disconnect();
    }
}

Example 47 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KeyedWindowedOperatorBenchmarkAppTest method test.

@Test
public void test() throws Exception {
    Configuration conf = new Configuration(false);
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    super.populateDAG(dag, conf);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.run(3000000);
    lc.shutdown();
}

Example 48 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class HBaseScanOperatorTest method testScan.

@Test
public void testScan() {
    try {
        HBaseTestHelper.populateHBase();
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        dag.setAttribute(DAG.APPLICATION_NAME, "HBaseScanOperatorTest");
        TestHBaseScanOperator thop = dag.addOperator("testhbasescan", TestHBaseScanOperator.class);
        HBaseTupleCollector tc = dag.addOperator("tuplecollector", HBaseTupleCollector.class);
        dag.addStream("ss", thop.outputPort, tc.inputPort);
        thop.getStore().setTableName("table1");
        thop.getStore().setZookeeperQuorum("127.0.0.1");
        thop.getStore().setZookeeperClientPort(2181);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        lc.run(30000);
        /*
      tuples = new ArrayList<HBaseTuple>();
      TestHBaseScanOperator thop = new TestHBaseScanOperator();
           thop.setTableName("table1");
      thop.setZookeeperQuorum("127.0.0.1");
      thop.setZookeeperClientPort(2822);
      thop.setupConfiguration();

      thop.emitTuples();
      */
        // TODO review the generated test code and remove the default call to fail.
        // fail("The test case is a prototype.");
        // Check total number
        List<HBaseTuple> tuples = HBaseTupleCollector.tuples;
        Assert.assertTrue(tuples.size() > 0);
        HBaseTuple tuple = HBaseTestHelper.findTuple(tuples, "row0", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-0");
        tuple = HBaseTestHelper.findTuple(tuples, "row499", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row499");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-499-0");
    } catch (Exception ex) {
        logger.error(ex.getMessage());
        assert false;
    }
}

Example 49 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class HBaseTransactionalAppendOperatorTest method testAppend.

@Test
public void testAppend() {
    try {
        HBaseTestHelper.startLocalCluster();
        HBaseTestHelper.clearHBase();
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        dag.setAttribute(DAG.APPLICATION_NAME, "HBaseAppendOperatorTest");
        HBaseColTupleGenerator ctg = dag.addOperator("coltuplegenerator", HBaseColTupleGenerator.class);
        TestHBaseAppendOperator thop = dag.addOperator("testhbaseput", TestHBaseAppendOperator.class);
        dag.addStream("ss", ctg.outputPort, thop.input);
        thop.getStore().setTableName("table1");
        thop.getStore().setZookeeperQuorum("127.0.0.1");
        thop.getStore().setZookeeperClientPort(2181);
        final LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        lc.run(30000);
        HBaseTuple tuple = HBaseTestHelper.getHBaseTuple("row0", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-0");
        tuple = HBaseTestHelper.getHBaseTuple("row0", "colfam0", "col-499");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-499");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-499");
    } catch (Exception ex) {
        logger.error(ex.getMessage());
        assert false;
    }
}

Example 50 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class HBaseGetOperatorTest method testGet.

@Test
public void testGet() {
    try {
        // HBaseTestHelper.startLocalCluster();
        HBaseTestHelper.populateHBase();
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        dag.setAttribute(DAG.APPLICATION_NAME, "HBaseGetOperatorTest");
        TestHBaseGetOperator thop = dag.addOperator("testhbaseget", TestHBaseGetOperator.class);
        HBaseStore store = new HBaseStore();
        thop.setStore(store);
        HBaseTupleCollector tc = dag.addOperator("tuplecollector", HBaseTupleCollector.class);
        dag.addStream("ss", thop.outputPort, tc.inputPort);
        thop.getStore().setTableName("table1");
        thop.getStore().setZookeeperQuorum("127.0.0.1");
        thop.getStore().setZookeeperClientPort(2181);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        lc.run(30000);
        /*
      tuples = new ArrayList<HBaseTuple>();
      TestHBaseGetOperator thop = new TestHBaseGetOperator();
           thop.setTableName("table1");
      thop.setZookeeperQuorum("127.0.0.1");
      thop.setZookeeperClientPort(2822);
      thop.setupConfiguration();

      thop.emitTuples();
      */
        // TODO review the generated test code and remove the default call to fail.
        // fail("The test case is a prototype.");
        // Check total number
        List<HBaseTuple> tuples = HBaseTupleCollector.tuples;
        Assert.assertTrue(tuples.size() > 0);
        HBaseTuple tuple = HBaseTestHelper.findTuple(tuples, "row0", "colfam0", "col-0");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-0");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-0");
        Assert.assertTrue(tuples.size() >= 499);
        tuple = HBaseTestHelper.findTuple(tuples, "row0", "colfam0", "col-499");
        Assert.assertNotNull("Tuple", tuple);
        Assert.assertEquals("Tuple row", tuple.getRow(), "row0");
        Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
        Assert.assertEquals("Tuple column name", tuple.getColName(), "col-499");
        Assert.assertEquals("Tuple column value", tuple.getColValue(), "val-0-499");
    } catch (Exception ex) {
        ex.printStackTrace();
        // logger.error(ex.getMessage());
        assert false;
    }
}