Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 26 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class MqttOutputOperatorTest method testDag.

@Test
public void testDag() throws Exception {
    String host = "localhost";
    int port = 1883;
    MqttClientConfig config = new MqttClientConfig();
    config.setHost(host);
    config.setPort(port);
    config.setCleanSession(true);
    sendingData.put("testa", "2");
    sendingData.put("testb", "20");
    sendingData.put("testc", "1000");
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    SourceModule source = dag.addOperator("source", SourceModule.class);
    TestMqttOutputOperator producer = dag.addOperator("producer", new TestMqttOutputOperator());
    producer.setMqttClientConfig(config);
    dag.addStream("Stream", source.outPort, producer.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    TestMqttOutputOperator.GetDataThread consumer = producer.new GetDataThread();
    producer.setup(null);
    consumer.start();
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    Thread.sleep(2000);
    lc.shutdown();
    Assert.assertEquals("emitted value for testNum was ", 3, receivedData.size());
    for (Map.Entry<String, String> e : receivedData.entrySet()) {
        if (e.getKey().equals("testa")) {
            Assert.assertEquals("emitted value for 'testa' was ", "2", e.getValue());
        } else if (e.getKey().equals("testb")) {
            Assert.assertEquals("emitted value for 'testb' was ", "20", e.getValue());
        } else if (e.getKey().equals("testc")) {
            Assert.assertEquals("emitted value for 'testc' was ", "1000", e.getValue());
        }
    }
    logger.debug("end of test");
}

Example 27 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaInputOperatorTest method testInputOperator.

public void testInputOperator(boolean hasFailure, boolean idempotent) throws Exception {
    // each broker should get a END_TUPLE message
    latch = new CountDownLatch(countDownAll ? totalCount + totalBrokers : totalBrokers);
    logger.info("Test Case: name: {}; totalBrokers: {}; hasFailure: {}; hasMultiCluster: {};" + " hasMultiPartition: {}, partition: {}", testName, totalBrokers, hasFailure, hasMultiCluster, hasMultiPartition, partition);
    // Start producer
    KafkaTestProducer p = new KafkaTestProducer(testName, hasMultiPartition, hasMultiCluster);
    p.setSendCount(totalCount);
    Thread t = new Thread(p);
    t.start();
    int expectedReceiveCount = totalCount + totalBrokers;
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortInputOperator node = dag.addOperator("Kafka input" + testName, KafkaSinglePortInputOperator.class);
    node.setInitialPartitionCount(1);
    // set topic
    node.setTopics(testName);
    node.setInitialOffset(AbstractKafkaInputOperator.InitialOffset.EARLIEST.name());
    node.setClusters(getClusterConfig());
    node.setStrategy(partition);
    if (idempotent) {
        node.setWindowDataManager(new FSWindowDataManager());
    }
    // Create Test tuple collector
    CollectorModule collector = dag.addOperator("TestMessageCollector", CollectorModule.class);
    collector.isIdempotentTest = idempotent;
    // Connect ports
    dag.addStream("Kafka message" + testName, node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    if (hasFailure) {
        setupHasFailureTest(node, dag);
    }
    // Create local cluster
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    // let the Controller to run the inside another thread. It is almost same as call Controller.runAsync(),
    // but Controller.runAsync() don't expose the thread which run it,
    // so we don't know when the thread will be terminated.
    // create this thread and then call join() to make sure the Controller shutdown completely.
    monitorThread = new Thread((StramLocalCluster) lc, "master");
    monitorThread.start();
    boolean notTimeout = true;
    try {
        // Wait 60s for consumer finish consuming all the messages
        notTimeout = latch.await(waitTime, TimeUnit.MILLISECONDS);
        lc.shutdown();
        // wait until control thread finished.
        monitorThread.join();
    } catch (Exception e) {
        logger.warn(e.getMessage());
    }
    t.join();
    if (!notTimeout || expectedReceiveCount != tupleCollection.size()) {
        logger.info("Number of received/expected tuples: {}/{}, testName: {}, tuples: \n{}", tupleCollection.size(), expectedReceiveCount, testName, tupleCollection);
    }
    Assert.assertTrue("TIMEOUT. testName: " + this.testName + "; Collected data: " + tupleCollection, notTimeout);
    // Check results
    Assert.assertTrue("testName: " + testName + "; Collected tuple size: " + tupleCollection.size() + "; Expected tuple size: " + expectedReceiveCount + "; data: \n" + tupleCollection, expectedReceiveCount == tupleCollection.size());
    logger.info("End of test case: {}", testName);
}

Example 28 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaInputOperatorTest method testKafkaInputOperator.

/**
 * Test AbstractKafkaSinglePortInputOperator (i.e. an input adapter for
 * Kafka, aka consumer). This module receives data from an outside test
 * generator through Kafka message bus and feed that data into Malhar
 * streaming platform.
 *
 * [Generate message and send that to Kafka message bus] ==> [Receive that
 * message through Kafka input adapter(i.e. consumer) and send using
 * emitTuples() interface on output port during onMessage call]
 *
 * @throws Exception
 */
public void testKafkaInputOperator(int sleepTime, final int totalCount, KafkaConsumer consumer, boolean isValid, boolean idempotent) throws Exception {
    // initial the latch for this test
    latch = new CountDownLatch(1);
    // Start producer
    KafkaTestProducer p = new KafkaTestProducer(TEST_TOPIC);
    p.setSendCount(totalCount);
    new Thread(p).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortStringInputOperator node = dag.addOperator("Kafka message consumer", KafkaSinglePortStringInputOperator.class);
    if (isSuicide) {
        // make some extreme assumptions to make it fail if checkpointing wrong offsets
        dag.setAttribute(Context.DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
        dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new FSStorageAgent("target/ck", new Configuration()));
        node.setMaxTuplesPerWindow(500);
    }
    if (idempotent) {
        node.setWindowDataManager(new FSWindowDataManager());
    }
    consumer.setTopic(TEST_TOPIC);
    node.setConsumer(consumer);
    consumer.setCacheSize(5000);
    if (isValid) {
        node.setZookeeper("localhost:" + KafkaOperatorTestBase.TEST_ZOOKEEPER_PORT[0]);
    }
    // Create Test tuple collector
    CollectorModule<String> collector = dag.addOperator("TestMessageCollector", new CollectorModule<String>());
    // Connect ports
    dag.addStream("Kafka message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    // Wait 30s for consumer finish consuming all the messages
    Assert.assertTrue("TIMEOUT: 30s ", latch.await(300000, TimeUnit.MILLISECONDS));
    // Check results
    Assert.assertTrue("Expected count >= " + totalCount + "; Actual count " + tupleCount.intValue(), totalCount <= tupleCount.intValue());
    logger.debug(String.format("Number of emitted tuples: %d", tupleCount.intValue()));
    p.close();
    lc.shutdown();
}

Example 29 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaOutputOperatorTest method testPOJOKafkaOutputOperator.

/**
 * Test AbstractKafkaOutputOperator (i.e. an output adapter for Kafka, aka producer).
 * This module sends data into kafka message bus.
 *
 * [Generate tuple] ==> [send tuple through Kafka output adapter(i.e. producer) into Kafka message bus]
 * ==> [receive data in outside Kaka listener (i.e consumer)]
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testPOJOKafkaOutputOperator() throws Exception {
    tupleCount = 0;
    // initialize the latch to synchronize the threads
    latch = new CountDownLatch(maxTuple);
    // Setup a message listener to receive the message
    KafkaTestConsumer listener = new KafkaTestConsumer("topic1");
    listener.setLatch(latch);
    new Thread(listener).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    StringGeneratorInputOperator generator = dag.addOperator("TestStringGenerator", StringGeneratorInputOperator.class);
    POJOKafkaOutputOperator node = dag.addOperator("KafkaMessageProducer", POJOKafkaOutputOperator.class);
    Properties props = new Properties();
    props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
    props.setProperty("producer.type", "async");
    props.setProperty("queue.buffering.max.ms", "200");
    props.setProperty("queue.buffering.max.messages", "10");
    node.setConfigProperties(props);
    node.setTopic("topic1");
    node.setBrokerList("localhost:9092");
    node.setBatchSize(5);
    // Connect ports
    dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
    Configuration conf = new Configuration(false);
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    // Immediately return unless latch timeout in 20 seconds
    latch.await(20, TimeUnit.SECONDS);
    lc.shutdown();
    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", maxTuple, listener.holdingBuffer.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.getMessage(listener.holdingBuffer.peek()));
    listener.close();
}

Example 30 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaOutputOperatorTest method testKafkaOutputOperator.

// End of StringGeneratorInputOperator
/**
 * Test AbstractKafkaOutputOperator (i.e. an output adapter for Kafka, aka producer).
 * This module sends data into an ActiveMQ message bus.
 *
 * [Generate tuple] ==> [send tuple through Kafka output adapter(i.e. producer) into Kafka message bus]
 * ==> [receive data in outside Kaka listener (i.e consumer)]
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testKafkaOutputOperator() throws Exception {
    // initialize the latch to synchronize the threads
    latch = new CountDownLatch(maxTuple);
    // Setup a message listener to receive the message
    KafkaTestConsumer listener = new KafkaTestConsumer("topic1");
    listener.setLatch(latch);
    new Thread(listener).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    // Create ActiveMQStringSinglePortOutputOperator
    StringGeneratorInputOperator generator = dag.addOperator("TestStringGenerator", StringGeneratorInputOperator.class);
    KafkaSinglePortOutputOperator node = dag.addOperator("KafkaMessageProducer", KafkaSinglePortOutputOperator.class);
    Properties props = new Properties();
    props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
    props.put("metadata.broker.list", "invalidhost:9092");
    props.setProperty("producer.type", "async");
    props.setProperty("queue.buffering.max.ms", "200");
    props.setProperty("queue.buffering.max.messages", "10");
    props.setProperty("batch.num.messages", "5");
    node.setConfigProperties(props);
    node.setTopic("topic1");
    // Connect ports
    dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // MLHR-1143: verify we can set broker list (and other properties) through configuration
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.KafkaMessageProducer.prop.configProperties(metadata.broker.list)", "localhost:9092");
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    // Immediately return unless latch timeout in 5 seconds
    latch.await(15, TimeUnit.SECONDS);
    lc.shutdown();
    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", tupleCount, listener.holdingBuffer.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.getMessage(listener.holdingBuffer.peek()));
    listener.close();
}