Examples with StreamMeta com.datatorrent.api.DAG.StreamMeta used on opensource projects

Example 11 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPartitionedPersistOperator.

@Test
public void testPartitionedPersistOperator() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PartitionedTestOperatorWithFiltering passThru = dag.addOperator("partition", new PartitionedTestOperatorWithFiltering());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input);
    dag.setInputPortAttribute(passThru.input, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.setInputPortAttribute(console1.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
    dag.addStream("Stream2", passThru.output, console.inport);
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 100000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if (console1.results.size() < 6) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        // Values as per persist operator's partition keys should be picked up
        Integer[] expectedResult = { 0, 4, 8, 12, 16, 20 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(console1.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], console1.results.get(i));
        }
    } finally {
        console1.results.clear();
        console.results.clear();
    }
}

Example 12 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOnSingleSinkWithFilteringContainerLocal.

@Test
public void testPersistStreamOnSingleSinkWithFilteringContainerLocal() throws ClassNotFoundException, IOException, InterruptedException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PassThruOperatorWithCodec passThru = dag.addOperator("PassThrough", new PassThruOperatorWithCodec(2));
    PassThruOperatorWithCodec passThru2 = dag.addOperator("Multiples_of_3", new PassThruOperatorWithCodec(3));
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final TestReceiverOperator console1 = dag.addOperator("console1", new TestReceiverOperator());
    TestPersistanceOperator persister = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input, passThru2.input).setLocality(Locality.CONTAINER_LOCAL);
    s.persistUsing("persister", persister, persister.inport);
    dag.addStream("Stream2", passThru.output, console.inport);
    dag.addStream("Stream3", passThru2.output, console1.inport);
    // runLocalClusterAndValidate(dag, console, persister);
    persister.results.clear();
    console.results.clear();
    console1.results.clear();
    // Validate union of results is received on persist operator
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 1000000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if ((console.results.size() < 6) || (console.results.size() < 6)) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        Integer[] expectedResult = { 0, 2, 3, 4, 6, 8, 9, 10, 12 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(persister.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], persister.results.get(i));
        }
    } finally {
        persister.results.clear();
        console.results.clear();
        console1.results.clear();
    }
}

Example 13 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenAllSinksAreRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenAllSinksAreRemoved() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    // Check stream persister is added to the dag
    Assert.assertNotNull("Stream persister operator should be present", dag.getOperatorMeta("Stream1_persister"));
    // remove sink operators x1, x2, x3 from dag and check that persister
    // operator is removed
    dag.removeOperator(x1);
    dag.removeOperator(x2);
    dag.removeOperator(x3);
    Assert.assertNull("Persister operator should have been removed after all sinks are removed", dag.getOperatorMeta("Stream1_persister"));
}

Example 14 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorGeneratesIdenticalOutputAsSink.

@Test
public void testPersistStreamOperatorGeneratesIdenticalOutputAsSink() throws ClassNotFoundException, IOException, InterruptedException {
    AscendingNumbersOperator input1 = dag.addOperator("input1", AscendingNumbersOperator.class);
    // Add PersistOperator directly to dag
    final TestReceiverOperator x = dag.addOperator("x", new TestReceiverOperator());
    StreamMeta stream = dag.addStream("Stream1", input1.outputPort, x.inport);
    // Use an instance of PersistOperator to persist stream
    TestPersistanceOperator persister = new TestPersistanceOperator();
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    runLocalClusterAndValidate(dag, x, persister);
}

Example 15 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorGeneratesUnionOfAllSinksOutput.

@Test
public void testPersistStreamOperatorGeneratesUnionOfAllSinksOutput() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PassThruOperatorWithCodec passThru1 = dag.addOperator("PassThrough1", new PassThruOperatorWithCodec(2));
    PassThruOperatorWithCodec passThru2 = dag.addOperator("PassThrough2", new PassThruOperatorWithCodec(3));
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final TestReceiverOperator console1 = dag.addOperator("console1", new TestReceiverOperator());
    TestPersistanceOperator persister = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru1.input, passThru2.input);
    s.persistUsing("persister", persister, persister.inport);
    dag.addStream("Stream2", passThru1.output, console.inport);
    dag.addStream("Stream3", passThru2.output, console1.inport);
    persister.results.clear();
    console.results.clear();
    console1.results.clear();
    // Validate union of results is received on persist operator
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 10000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if ((console.results.size() < 6) || (console.results.size() < 6)) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        Integer[] expectedResult = { 0, 2, 3, 4, 6, 8, 9, 10, 12 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(persister.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], persister.results.get(i));
        }
    } finally {
        persister.results.clear();
        console.results.clear();
        console1.results.clear();
    }
}