Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 56 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class ShardManagerTest method testShardManager.

@Test
public void testShardManager() throws Exception {
    // Set to 3 because we want to make sure all the tuples from both 2 partitions are received and offsets has been updated to 102
    latch = new CountDownLatch(3);
    // Start producer
    KinesisTestProducer p = new KinesisTestProducer(streamName, true);
    p.setSendCount(totalCount);
    // wait the producer send all records
    p.run();
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    KinesisUtil.getInstance().setClient(client);
    // Create KinesisSinglePortStringInputOperator
    KinesisStringInputOperator node = dag.addOperator("Kinesis consumer", KinesisStringInputOperator.class);
    node.setAccessKey(credentials.getCredentials().getAWSSecretKey());
    node.setSecretKey(credentials.getCredentials().getAWSAccessKeyId());
    node.setStreamName(streamName);
    ShardManager tfm = new ShardManager();
    node.setShardManager(tfm);
    node.setStrategy(AbstractKinesisInputOperator.PartitionStrategy.MANY_TO_ONE.toString());
    node.setRepartitionInterval(-1);
    KinesisConsumer consumer = new KinesisConsumer();
    // set topic
    consumer.setStreamName(streamName);
    // set the brokerlist used to initialize the partition
    consumer.setInitialOffset("earliest");
    node.setConsumer(consumer);
    // Create Test tuple collector
    CollectorModule collector = dag.addOperator("RecordCollector", new CollectorModule());
    // Connect ports
    dag.addStream("Kinesis Records", node.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(true);
    lc.runAsync();
    // Wait 15s for consumer finish consuming all the records
    latch.await(10000, TimeUnit.MILLISECONDS);
    assertEquals("ShardPos Size", 2, node.getShardManager().loadInitialShardPositions().size());
    Iterator ite = node.getShardManager().loadInitialShardPositions().entrySet().iterator();
    Entry e = (Entry) ite.next();
    assertNotEquals("Record Seq No in Shard Id 1", "", e.getValue());
    e = (Entry) ite.next();
    assertNotEquals("Record Seq No in Shard Id 2", "", e.getValue());
    // Check results
    assertEquals("Tuple count", totalCount, collectedTuples.size());
    logger.debug(String.format("Number of emitted tuples: %d -> %d", collectedTuples.size(), totalCount));
    lc.shutdown();
}

Example 57 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KafkaOutputOperatorTest method ReadFromKafka.

private List<Person> ReadFromKafka() {
    tupleCollection.clear();
    // Create KafkaSinglePortStringInputOperator
    Properties props = new Properties();
    props.put(BOOTSTRAP_SERVERS_CONFIG, getClusterConfig());
    props.put(KEY_DESERIALIZER_CLASS_CONFIG, KafkaSinglePortExactlyOnceOutputOperator.KEY_DESERIALIZER);
    props.put(VALUE_DESERIALIZER_CLASS_CONFIG, VALUE_DESERIALIZER);
    props.put(GROUP_ID_CONFIG, "KafkaTest");
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    // Create KafkaSinglePortStringInputOperator
    KafkaSinglePortInputOperator node = dag.addOperator("Kafka input", KafkaSinglePortInputOperator.class);
    node.setConsumerProps(props);
    node.setInitialPartitionCount(1);
    // set topic
    node.setTopics(testName);
    node.setInitialOffset(AbstractKafkaInputOperator.InitialOffset.EARLIEST.name());
    node.setClusters(getClusterConfig());
    node.setStrategy("one_to_one");
    // Create Test tuple collector
    CollectorModule collector1 = dag.addOperator("collector", new CollectorModule());
    // Connect ports
    dag.addStream("Kafka message", node.outputPort, collector1.inputPort);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.run(30000);
    return tupleCollection;
}

Example 58 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class FunctionOperatorTest method testFilterOperator.

@Test
public void testFilterOperator() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    FunctionOperator.FilterFunctionOperator<Integer> filter0 = new FunctionOperator.FilterFunctionOperator<Integer>(new Function.FilterFunction<Integer>() {

        @Override
        public boolean f(Integer in) {
            return in % divider == 0;
        }
    });
    NumberGenerator numGen = dag.addOperator("numGen", new NumberGenerator());
    FunctionOperator.FilterFunctionOperator<Integer> filter = dag.addOperator("filter", filter0);
    ResultCollector collector = dag.addOperator("collector", new ResultCollector());
    dag.addStream("raw numbers", numGen.output, filter.input);
    dag.addStream("filtered results", filter.output, collector.input);
    // Create local cluster
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return TupleCount == NumTuples / divider;
        }
    });
    lc.run(5000);
    Assert.assertEquals(sum, 20);
}

Example 59 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class FunctionOperatorTest method testFlatMapOperator.

@Test
public void testFlatMapOperator() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    NumberListGenerator numGen = dag.addOperator("numGen", new NumberListGenerator());
    FunctionOperator.FlatMapFunctionOperator<List<Integer>, Integer> fm = dag.addOperator("flatmap", new FunctionOperator.FlatMapFunctionOperator<>(new FmFunction()));
    ResultCollector collector = dag.addOperator("collector", new ResultCollector());
    dag.addStream("raw numbers", numGen.output, fm.input);
    dag.addStream("flatmap results", fm.output, collector.input);
    // Create local cluster
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return TupleCount == 13;
        }
    });
    lc.run(5000);
    Assert.assertEquals(sum, 39555);
}

Example 60 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class FunctionOperatorTest method testMapOperator.

@Test
public void testMapOperator() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    NumberGenerator numGen = dag.addOperator("numGen", new NumberGenerator());
    FunctionOperator.MapFunctionOperator<Integer, Integer> mapper = dag.addOperator("mapper", new FunctionOperator.MapFunctionOperator<Integer, Integer>(new Square()));
    ResultCollector collector = dag.addOperator("collector", new ResultCollector());
    dag.addStream("raw numbers", numGen.output, mapper.input);
    dag.addStream("mapped results", mapper.output, collector.input);
    // Create local cluster
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return TupleCount == NumTuples;
        }
    });
    lc.run(5000);
    Assert.assertEquals(sum, 285);
}