Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 51 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaExactlyOnceOutputOperatorTest method testKafkaExactOnceOutputOperator.

// End of StringGeneratorInputOperator
/**
 * Test AbstractKafkaExactOnceOutputOperator (i.e. an output adapter for Kafka, aka producer).
 * This module sends data into a Kafka message bus.
 *
 * [Generate tuple] ==> [send tuple through Kafka output adapter(i.e. producer) into Kafka message bus](fail the producer at certain point and bring it back)
 * ==> [receive data in outside Kafka listener (i.e consumer)] ==> Verify kafka doesn't receive duplicated message
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "rawtypes" })
public void testKafkaExactOnceOutputOperator() throws Exception {
    // initialize the latch to synchronize the threads
    latch = new CountDownLatch(maxTuple);
    // Setup a message listener to receive the message
    KafkaTestConsumer listener = new KafkaTestConsumer("topic1");
    listener.setLatch(latch);
    // Malhar module to send message
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    final DAG dag = lma.getDAG();
    StringGeneratorInputOperator generator = dag.addOperator("TestStringGenerator", StringGeneratorInputOperator.class);
    final SimpleKafkaExactOnceOutputOperator node = dag.addOperator("Kafka message producer", SimpleKafkaExactOnceOutputOperator.class);
    Properties props = new Properties();
    props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
    props.put("metadata.broker.list", "localhost:9092");
    props.setProperty("producer.type", "async");
    props.setProperty("queue.buffering.max.ms", "200");
    props.setProperty("queue.buffering.max.messages", "10");
    props.setProperty("batch.num.messages", "5");
    node.setConfigProperties(props);
    // Set configuration parameters for Kafka
    node.setTopic("topic1");
    // Connect ports
    dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    Future f = Executors.newFixedThreadPool(1).submit(listener);
    f.get(30, TimeUnit.SECONDS);
    lc.shutdown();
    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", maxTuple, listener.holdingBuffer.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.getMessage(listener.holdingBuffer.peek()));
    listener.close();
}

Example 52 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaPartitionableInputOperatorTest method testPartitionableInputOperator.

public void testPartitionableInputOperator(KafkaConsumer consumer) throws Exception {
    // each broker should get a END_TUPLE message
    latch = new CountDownLatch(totalBrokers);
    int totalCount = 10000;
    // Start producer
    KafkaTestProducer p = new KafkaTestProducer(TEST_TOPIC, hasMultiPartition, hasMultiCluster);
    p.setSendCount(totalCount);
    new Thread(p).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortStringInputOperator node = dag.addOperator("Kafka message consumer", KafkaSinglePortStringInputOperator.class);
    node.setInitialPartitionCount(1);
    // set topic
    consumer.setTopic(TEST_TOPIC);
    consumer.setInitialOffset("earliest");
    node.setConsumer(consumer);
    String clusterString = "cluster1::localhost:" + TEST_ZOOKEEPER_PORT[0] + (hasMultiCluster ? ";cluster2::localhost:" + TEST_ZOOKEEPER_PORT[1] : "");
    node.setZookeeper(clusterString);
    // Create Test tuple collector
    CollectorModule<String> collector = dag.addOperator("TestMessageCollector", new CollectorModule<String>());
    // Connect ports
    dag.addStream("Kafka message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    // Wait 30s for consumer finish consuming all the messages
    Assert.assertTrue("TIMEOUT: 40s ", latch.await(40000, TimeUnit.MILLISECONDS));
    // Check results
    Assert.assertEquals("Collections size", 1, collections.size());
    Assert.assertEquals("Tuple count", totalCount, collections.get(collector.inputPort.id).size());
    logger.debug(String.format("Number of emitted tuples: %d", collections.get(collector.inputPort.id).size()));
    p.close();
    lc.shutdown();
    // kafka has a bug shutdown connector you have to make sure kafka client resource has been cleaned before clean the broker
    Thread.sleep(5000);
}

Example 53 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KinesisInputOperatorTest method testWindowDataManager.

@Test
public void testWindowDataManager() throws Exception {
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    KinesisStringInputOperator inputOperator = dag.addOperator("KinesisInput", new KinesisStringInputOperator() {

        @Override
        public void deactivate() {
        }

        @Override
        public void teardown() {
        }
    });
    testMeta.operator = inputOperator;
    Assert.assertTrue("Default behaviour of WindowDataManager changed", (inputOperator.getWindowDataManager() instanceof WindowDataManager.NoopWindowDataManager));
}

Example 54 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KinesisInputOperatorTest method testKinesisByteArrayInputOperator.

@Test
public void testKinesisByteArrayInputOperator() throws Exception {
    int totalCount = 10;
    // initial the latch for this test
    latch = new CountDownLatch(1);
    // Start producer
    KinesisTestProducer p = new KinesisTestProducer(streamName);
    p.setSendCount(totalCount);
    p.setBatchSize(9);
    new Thread(p).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KinesisByteArrayInputOperator and set some properties with respect to consumer.
    KinesisByteArrayInputOperator node = dag.addOperator("Kinesis message consumer", KinesisByteArrayInputOperator.class);
    node.setAccessKey(credentials.getCredentials().getAWSSecretKey());
    node.setSecretKey(credentials.getCredentials().getAWSAccessKeyId());
    KinesisConsumer consumer = new KinesisConsumer();
    consumer.setStreamName(streamName);
    consumer.setRecordsLimit(totalCount);
    node.setConsumer(consumer);
    // Create Test tuple collector
    CollectorModule<byte[]> collector = dag.addOperator("TestMessageCollector", new CollectorModule<byte[]>());
    // Connect ports
    dag.addStream("Kinesis message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    // Wait 45s for consumer finish consuming all the messages
    latch.await(45000, TimeUnit.MILLISECONDS);
    // Check results
    Assert.assertEquals("Collections size", 1, collections.size());
    Assert.assertEquals("Tuple count", totalCount, collections.get(collector.inputPort.id).size());
    logger.debug(String.format("Number of emitted tuples: %d", collections.get(collector.inputPort.id).size()));
    lc.shutdown();
}

Example 55 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KinesisInputOperatorTest method testKinesisInputOperator.

/**
 * Test AbstractKinesisSinglePortInputOperator (i.e. an input adapter for
 * Kinesis, consumer). This module receives data from an outside test
 * generator through Kinesis message bus and feed that data into Malhar
 * streaming platform.
 *
 * [Generate message and send that to Kinesis message bus] ==> [Receive that
 * message through Kinesis input adapter(i.e. consumer) and send using
 * emitTuples() interface on output port during onMessage call]
 *
 * @throws Exception
 */
@Test
public void testKinesisInputOperator() throws Exception {
    int totalCount = 100;
    // initial the latch for this test
    latch = new CountDownLatch(1);
    // Start producer
    KinesisTestProducer p = new KinesisTestProducer(streamName);
    p.setSendCount(totalCount);
    p.setBatchSize(500);
    new Thread(p).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KinesisSinglePortStringInputOperator
    KinesisStringInputOperator node = dag.addOperator("Kinesis message consumer", KinesisStringInputOperator.class);
    node.setAccessKey(credentials.getCredentials().getAWSSecretKey());
    node.setSecretKey(credentials.getCredentials().getAWSAccessKeyId());
    KinesisConsumer consumer = new KinesisConsumer();
    consumer.setStreamName(streamName);
    consumer.setRecordsLimit(totalCount);
    node.setConsumer(consumer);
    // Create Test tuple collector
    CollectorModule<String> collector = dag.addOperator("TestMessageCollector", new CollectorModule<String>());
    // Connect ports
    dag.addStream("Kinesis message", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    // Wait 45s for consumer finish consuming all the messages
    latch.await(45000, TimeUnit.MILLISECONDS);
    // Check results
    Assert.assertEquals("Collections size", 1, collections.size());
    Assert.assertEquals("Tuple count", totalCount, collections.get(collector.inputPort.id).size());
    logger.debug(String.format("Number of emitted tuples: %d", collections.get(collector.inputPort.id).size()));
    lc.shutdown();
}