Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 41 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class MqttInputOperatorTest method testInputOperator.

@Test
public void testInputOperator() throws InterruptedException, Exception {
    String host = "localhost";
    int port = 1883;
    MqttClientConfig config = new MqttClientConfig();
    config.setHost(host);
    config.setPort(port);
    config.setCleanSession(true);
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    final TestMqttInputOperator input = dag.addOperator("input", TestMqttInputOperator.class);
    CollectorModule<KeyValPair<String, String>> collector = dag.addOperator("collector", new CollectorModule<KeyValPair<String, String>>());
    input.addSubscribeTopic("a", QoS.AT_MOST_ONCE);
    input.addSubscribeTopic("b", QoS.AT_MOST_ONCE);
    input.addSubscribeTopic("c", QoS.AT_MOST_ONCE);
    input.setMqttClientConfig(config);
    input.setup(null);
    dag.addStream("stream", input.outputPort, collector.inputPort);
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    long timeout = System.currentTimeMillis() + 3000;
    while (true) {
        if (activated) {
            break;
        }
        Assert.assertTrue("Activation timeout", timeout > System.currentTimeMillis());
        Thread.sleep(1000);
    }
    input.activate(null);
    // Thread.sleep(3000);
    input.generateData();
    long timeout1 = System.currentTimeMillis() + 5000;
    try {
        while (true) {
            if (resultCount == 0) {
                Thread.sleep(10);
                Assert.assertTrue("timeout without getting any data", System.currentTimeMillis() < timeout1);
            } else {
                break;
            }
        }
    } catch (InterruptedException ex) {
    // ignore
    }
    lc.shutdown();
    Assert.assertEquals("Number of emitted tuples", 3, resultMap.size());
    Assert.assertEquals("value of a is ", "10", resultMap.get("a"));
    Assert.assertEquals("value of b is ", "200", resultMap.get("b"));
    Assert.assertEquals("value of c is ", "3000", resultMap.get("c"));
    logger.debug("resultCount:" + resultCount);
}

Example 42 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class RabbitMQInputOperatorTest method runTest.

protected void runTest(final int testNum) throws IOException {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    RabbitMQInputOperator consumer = dag.addOperator("Consumer", RabbitMQInputOperator.class);
    consumer.setWindowDataManager(new FSWindowDataManager());
    final CollectorModule<byte[]> collector = dag.addOperator("Collector", new CollectorModule<byte[]>());
    consumer.setHost("localhost");
    consumer.setExchange("testEx");
    consumer.setExchangeType("fanout");
    final RabbitMQMessageGenerator publisher = new RabbitMQMessageGenerator();
    publisher.setup();
    dag.addStream("Stream", consumer.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 100000L;
            try {
                while (!collector.inputPort.collections.containsKey("collector") && System.currentTimeMillis() - startTms < timeout) {
                    Thread.sleep(500);
                }
                publisher.generateMessages(testNum);
                startTms = System.currentTimeMillis();
                while (System.currentTimeMillis() - startTms < timeout) {
                    List<?> list = collector.inputPort.collections.get("collector");
                    if (list.size() < testNum * 3) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (IOException ex) {
                logger.error(ex.getMessage(), ex);
                DTThrowable.rethrow(ex);
            } catch (InterruptedException ex) {
                DTThrowable.rethrow(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    logger.debug("collection size: {} {}", collector.inputPort.collections.size(), collector.inputPort.collections);
    MessageQueueTestHelper.validateResults(testNum, collector.inputPort.collections);
}

Example 43 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class RomeSyndicationOperatorTest method testRun.

/**
 * Test of run method, of class RomeSyndicationOperator.
 */
@Test
public void testRun() {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    RomeSyndicationOperator rop = dag.addOperator("romesyndication", RomeSyndicationOperator.class);
    FeedCollector fc = dag.addOperator("feedcollector", FeedCollector.class);
    dag.addStream("ss", rop.outputPort, fc.input);
    rop.setInterval(2000);
    // rop.setLocation("http://rss.cnn.com/rss/cnn_topstories.rss");
    rop.setStreamProvider(new CNNSyndicationTestProvider());
    try {
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        lc.run(10000);
        Assert.assertEquals("Entries size", entries.size(), 10);
        // Check first entry
        Assert.assertEquals("First entry title", entries.get(0).getSyndEntry().getTitle(), "Dimensions Computation (Aggregate Navigator) Part 1: Intro");
        Assert.assertEquals("First entry URI", entries.get(0).getSyndEntry().getUri(), "https://www.datatorrent.com/?p=2399");
        // Check first entry from second run
        Assert.assertEquals("Second run first entry title", entries.get(7).getSyndEntry().getTitle(), "Building Applications with Apache Apex and Malhar");
        Assert.assertEquals("Second run first entry URI", entries.get(7).getSyndEntry().getUri(), "https://www.datatorrent.com/?p=2054");
        // Check last entry
        Assert.assertEquals("Last entry title", entries.get(9).getSyndEntry().getTitle(), "Dimensions Computation (Aggregate Navigator) Part 2: Implementation");
        Assert.assertEquals("Last entry URI", entries.get(9).getSyndEntry().getUri(), "https://www.datatorrent.com/?p=2401");
    } catch (Exception ex) {
        logger.error(ex.getMessage());
        Assert.assertFalse(true);
    }
}

Example 44 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class ZeroMQInputOperatorTest method testHelper.

protected void testHelper(final int testNum) {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    final ZeroMQMessageGenerator publisher = new ZeroMQMessageGenerator();
    publisher.setup();
    ZeroMQInputOperator generator = dag.addOperator("Generator", ZeroMQInputOperator.class);
    final CollectorModule<byte[]> collector = dag.addOperator("Collector", new CollectorModule<byte[]>());
    generator.setFilter("");
    generator.setUrl("tcp://localhost:5556");
    generator.setSyncUrl("tcp://localhost:5557");
    dag.addStream("Stream", generator.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    new Thread() {

        @Override
        public void run() {
            try {
                publisher.generateMessages(testNum);
            } catch (InterruptedException ex) {
                logger.debug(ex.toString());
            }
        }
    }.start();
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 10000L;
            try {
                while (!collector.inputPort.collections.containsKey("collector") && System.currentTimeMillis() - startTms < timeout) {
                    Thread.sleep(500);
                }
                Thread.sleep(1000);
                startTms = System.currentTimeMillis();
                while (System.currentTimeMillis() - startTms < timeout) {
                    List<?> list = collector.inputPort.collections.get("collector");
                    if (list.size() < testNum * 3) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (InterruptedException ex) {
                DTThrowable.rethrow(ex);
            } finally {
                logger.debug("Shutting down..");
                lc.shutdown();
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    DTThrowable.rethrow(e);
                } finally {
                    publisher.teardown();
                }
            }
        }
    }.start();
    lc.run();
    // logger.debug("collection size:"+collector.inputPort.collections.size()+" "+collector.inputPort.collections.toString());
    MessageQueueTestHelper.validateResults(testNum, collector.inputPort.collections);
    logger.debug("end of test");
}

Example 45 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class ZeroMQOutputOperatorTest method runTest.

protected void runTest(final int testNum) {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    SourceModule source = dag.addOperator("source", new SourceModule());
    source.setTestNum(testNum);
    final ZeroMQOutputOperator collector = dag.addOperator("generator", new ZeroMQOutputOperator());
    collector.setUrl("tcp://*:5556");
    collector.setSyncUrl("tcp://*:5557");
    collector.setSUBSCRIBERS_EXPECTED(1);
    dag.addStream("Stream", source.outPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    final ZeroMQMessageReceiver receiver = new ZeroMQMessageReceiver();
    receiver.setup();
    final Thread t = new Thread(receiver);
    t.start();
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            try {
                Thread.sleep(1000);
                long timeout = 10000L;
                long startTms = System.currentTimeMillis();
                while (System.currentTimeMillis() - startTms < timeout) {
                    if (receiver.count < testNum * 3) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (InterruptedException ex) {
                DTThrowable.rethrow(ex);
            } finally {
                logger.debug("done...");
                lc.shutdown();
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    DTThrowable.rethrow(e);
                } finally {
                    t.interrupt();
                    receiver.teardown();
                }
            }
        }
    }.start();
    lc.run();
    Assert.assertEquals("emitted value for testNum was ", testNum * 3, receiver.count);
    for (Map.Entry<String, Integer> e : receiver.dataMap.entrySet()) {
        if (e.getKey().equals("a")) {
            Assert.assertEquals("emitted value for 'a' was ", new Integer(2), e.getValue());
        } else if (e.getKey().equals("b")) {
            Assert.assertEquals("emitted value for 'b' was ", new Integer(20), e.getValue());
        } else if (e.getKey().equals("c")) {
            Assert.assertEquals("emitted value for 'c' was ", new Integer(1000), e.getValue());
        }
    }
}