Examples with StreamingApplication com.datatorrent.api.StreamingApplication used on opensource projects

Example 11 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-malhar by apache.

the class TestNiFiInputApplication method main.

public static void main(String[] args) throws Exception {
    StreamingApplication app = new TestNiFiInputApplication();
    LocalMode.runApp(app, new Configuration(false), 10000);
    Thread.sleep(2000);
    System.exit(0);
}

Example 12 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-malhar by apache.

the class SpillableBenchmarkAppTester method test.

@Test
public void test() throws Exception {
    Configuration conf = new Configuration(false);
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    super.populateDAG(dag, conf);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.run(600000);
    lc.shutdown();
    if (SpillableTestOperator.errorTrace != null) {
        logger.error("Error.", SpillableTestOperator.errorTrace);
        Assert.assertNull(SpillableTestOperator.errorTrace.getMessage(), SpillableTestOperator.errorTrace);
    }
}

Example 13 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-malhar by apache.

the class ManagedStateBenchmarkAppTest method test.

public void test(ExecMode exeMode) throws Exception {
    Configuration conf = new Configuration(false);
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    super.populateDAG(dag, conf);
    storeOperator.setExecMode(exeMode);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.run(3000000);
    lc.shutdown();
}

Example 14 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-malhar by apache.

the class KinesisOutputOperatorTest method testKinesisOutputOperator.

/**
 * Test AbstractKinesisOutputOperator (i.e. an output adapter for Kinesis, aka producer).
 * This module sends data into an ActiveMQ message bus.
 *
 * [Generate tuple] ==> [send tuple through Kinesis output adapter(i.e. producer) into Kinesis message bus]
 * ==> [receive data in outside Kinesis listener (i.e consumer)]
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "unchecked" })
public void testKinesisOutputOperator() throws Exception {
    // Setup a message listener to receive the message
    KinesisTestConsumer listener = null;
    if (enableConsumer) {
        listener = createConsumerListener(streamName);
        if (listener != null) {
            // initialize the latch to synchronize the threads
            doneLatch = new CountDownLatch(maxTuple);
            listener.setDoneLatch(doneLatch);
            listenerThread = new Thread(listener);
            listenerThread.start();
        }
    }
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    // Create ActiveMQStringSinglePortOutputOperator
    G generator = addGenerateOperator(dag);
    O node = addTestingOperator(dag);
    configureTestingOperator(node);
    // Connect ports
    dag.addStream("Kinesis message", getOutputPortOfGenerator(generator), node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    Configuration conf = new Configuration(false);
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    int waitTime = 300000;
    if (doneLatch != null) {
        doneLatch.await(waitTime, TimeUnit.MILLISECONDS);
    } else {
        try {
            Thread.sleep(waitTime);
        } catch (Exception e) {
        // 
        }
    }
    if (listener != null) {
        listener.setIsAlive(false);
    }
    if (listenerThread != null) {
        listenerThread.join(1000);
    }
    lc.shutdown();
    // Check values send vs received
    if (listener != null) {
        Assert.assertEquals("Number of emitted tuples", maxTuple, listener.holdingBuffer.size());
        logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    }
    if (listener != null) {
        listener.close();
    }
}

Example 15 with StreamingApplication

use of com.datatorrent.api.StreamingApplication in project apex-malhar by apache.

the class KafkaOutputOperatorTest method testKafkaOutputOperator.

// End of StringGeneratorInputOperator
/**
 * Test AbstractKafkaOutputOperator (i.e. an output adapter for Kafka, aka producer).
 * This module sends data into an ActiveMQ message bus.
 *
 * [Generate tuple] ==> [send tuple through Kafka output adapter(i.e. producer) into Kafka message bus]
 * ==> [receive data in outside Kaka listener (i.e consumer)]
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testKafkaOutputOperator() throws Exception {
    // initialize the latch to synchronize the threads
    latch = new CountDownLatch(maxTuple);
    // Setup a message listener to receive the message
    KafkaTestConsumer listener = new KafkaTestConsumer("topic1");
    listener.setLatch(latch);
    new Thread(listener).start();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    // Create ActiveMQStringSinglePortOutputOperator
    StringGeneratorInputOperator generator = dag.addOperator("TestStringGenerator", StringGeneratorInputOperator.class);
    KafkaSinglePortOutputOperator node = dag.addOperator("KafkaMessageProducer", KafkaSinglePortOutputOperator.class);
    Properties props = new Properties();
    props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
    props.put("metadata.broker.list", "invalidhost:9092");
    props.setProperty("producer.type", "async");
    props.setProperty("queue.buffering.max.ms", "200");
    props.setProperty("queue.buffering.max.messages", "10");
    props.setProperty("batch.num.messages", "5");
    node.setConfigProperties(props);
    node.setTopic("topic1");
    // Connect ports
    dag.addStream("Kafka message", generator.outputPort, node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // MLHR-1143: verify we can set broker list (and other properties) through configuration
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.KafkaMessageProducer.prop.configProperties(metadata.broker.list)", "localhost:9092");
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    // Immediately return unless latch timeout in 5 seconds
    latch.await(15, TimeUnit.SECONDS);
    lc.shutdown();
    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", tupleCount, listener.holdingBuffer.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.getMessage(listener.holdingBuffer.peek()));
    listener.close();
}