Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 31 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class KinesisOutputOperatorTest method testKinesisOutputOperator.

/**
 * Test AbstractKinesisOutputOperator (i.e. an output adapter for Kinesis, aka producer).
 * This module sends data into an ActiveMQ message bus.
 *
 * [Generate tuple] ==> [send tuple through Kinesis output adapter(i.e. producer) into Kinesis message bus]
 * ==> [receive data in outside Kinesis listener (i.e consumer)]
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "unchecked" })
public void testKinesisOutputOperator() throws Exception {
    // Setup a message listener to receive the message
    KinesisTestConsumer listener = null;
    if (enableConsumer) {
        listener = createConsumerListener(streamName);
        if (listener != null) {
            // initialize the latch to synchronize the threads
            doneLatch = new CountDownLatch(maxTuple);
            listener.setDoneLatch(doneLatch);
            listenerThread = new Thread(listener);
            listenerThread.start();
        }
    }
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    // Create ActiveMQStringSinglePortOutputOperator
    G generator = addGenerateOperator(dag);
    O node = addTestingOperator(dag);
    configureTestingOperator(node);
    // Connect ports
    dag.addStream("Kinesis message", getOutputPortOfGenerator(generator), node.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    Configuration conf = new Configuration(false);
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    int waitTime = 300000;
    if (doneLatch != null) {
        doneLatch.await(waitTime, TimeUnit.MILLISECONDS);
    } else {
        try {
            Thread.sleep(waitTime);
        } catch (Exception e) {
        // 
        }
    }
    if (listener != null) {
        listener.setIsAlive(false);
    }
    if (listenerThread != null) {
        listenerThread.join(1000);
    }
    lc.shutdown();
    // Check values send vs received
    if (listener != null) {
        Assert.assertEquals("Number of emitted tuples", maxTuple, listener.holdingBuffer.size());
        logger.debug(String.format("Number of emitted tuples: %d", listener.holdingBuffer.size()));
    }
    if (listener != null) {
        listener.close();
    }
}

Example 32 with DAG

use of com.datatorrent.api.DAG in project apex-malhar by apache.

the class HBasePOJOInputOperatorTest method test.

@Test
public void test() throws Exception {
    // Create DAG for testing.
    LocalMode lma = LocalMode.newInstance();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
        }
    };
    DAG dag = lma.getDAG();
    // Create ActiveMQStringSinglePortOutputOperator
    MyGenerator generator = dag.addOperator(OPERATOR.GENERATOR.name(), MyGenerator.class);
    generator.setTupleNum(TUPLE_NUM);
    hbaseOutputOperator = dag.addOperator(OPERATOR.HBASEOUTPUT.name(), hbaseOutputOperator);
    hbaseInputOperator = dag.addOperator(OPERATOR.HBASEINPUT.name(), hbaseInputOperator);
    dag.setOutputPortAttribute(hbaseInputOperator.outputPort, Context.PortContext.TUPLE_CLASS, TestPOJO.class);
    TupleCacheOutputOperator output = dag.addOperator(OPERATOR.OUTPUT.name(), TupleCacheOutputOperator.class);
    // Connect ports
    dag.addStream("queue1", generator.outputPort, hbaseOutputOperator.input).setLocality(DAG.Locality.NODE_LOCAL);
    dag.addStream("queue2", hbaseInputOperator.outputPort, output.inputPort).setLocality(DAG.Locality.NODE_LOCAL);
    Configuration conf = new Configuration(false);
    lma.prepareDAG(app, conf);
    // Create local cluster
    final LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    long start = System.currentTimeMillis();
    // generator.doneLatch.await();
    while (true) {
        Thread.sleep(1000);
        logger.info("Tuple row key: ", output.getReceivedTuples());
        logger.info("Received tuple number {}, instance is {}.", output.getReceivedTuples() == null ? 0 : output.getReceivedTuples().size(), System.identityHashCode(output));
        if (output.getReceivedTuples() != null && output.getReceivedTuples().size() == TUPLE_NUM) {
            break;
        }
        if (System.currentTimeMillis() - start > RUN_DURATION) {
            throw new RuntimeException("Testcase taking too long");
        }
    }
    lc.shutdown();
    validate(generator.getTuples(), output.getReceivedTuples());
}

Example 33 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testOperatorLevelProperties.

@Test
public void testOperatorLevelProperties() {
    String appName = "app1";
    final GenericTestOperator operator1 = new GenericTestOperator();
    final GenericTestOperator operator2 = new GenericTestOperator();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", operator1);
            dag.addOperator("operator2", operator2);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.myStringProperty", "pv1");
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    Assert.assertEquals("apv1", operator1.getMyStringProperty());
    Assert.assertEquals("pv1", operator2.getMyStringProperty());
    Assert.assertEquals(true, operator2.isBooleanProperty());
}

Example 34 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testPrepareDAG.

@Test
public void testPrepareDAG() {
    final MutableBoolean appInitialized = new MutableBoolean(false);
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            Assert.assertEquals("", "hostname:9090", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
            dag.setAttribute(DAG.GATEWAY_CONNECT_ADDRESS, "hostname:9091");
            appInitialized.setValue(true);
        }
    };
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = new LogicalPlan();
    pb.prepareDAG(dag, app, "testconfig");
    Assert.assertTrue("populateDAG called", appInitialized.booleanValue());
    Assert.assertEquals("populateDAG overrides attribute", "hostname:9091", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
}

Example 35 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testTestTupleClassAttrSetFromConfig.

@Test
public void testTestTupleClassAttrSetFromConfig() {
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.port.schemaRequiredPort.attr.TUPLE_CLASS", "com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest$TestSchema");
    StreamingApplication streamingApplication = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            TestGeneratorInputOperator o1 = dag.addOperator("o1", new TestGeneratorInputOperator());
            SchemaTestOperator o2 = dag.addOperator("o2", new SchemaTestOperator());
            dag.addStream("stream", o1.outport, o2.schemaRequiredPort);
        }
    };
    LogicalPlan dag = new LogicalPlan();
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    lpc.prepareDAG(dag, streamingApplication, "app");
    dag.validate();
}