Examples with StramLocalCluster com.datatorrent.stram.StramLocalCluster used on opensource projects

Example 1 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StreamPersistanceTests method runLocalClusterAndValidate.

private void runLocalClusterAndValidate(LogicalPlan dag, final TestReceiverOperator x, final TestPersistanceOperator persister) throws IOException, ClassNotFoundException {
    try {
        x.results.clear();
        persister.results.clear();
        // Run local cluster and verify both results are identical
        final StramLocalCluster lc = new StramLocalCluster(dag);
        new Thread("LocalClusterController") {

            @Override
            public void run() {
                long startTms = System.currentTimeMillis();
                long timeout = 100000L;
                try {
                    while (System.currentTimeMillis() - startTms < timeout) {
                        if (x.results.size() < 1000) {
                            Thread.sleep(10);
                        } else {
                            break;
                        }
                    }
                } catch (Exception ex) {
                    throw Throwables.propagate(ex);
                } finally {
                    lc.shutdown();
                }
            }
        }.start();
        lc.run();
        int maxTuples = x.results.size() > persister.results.size() ? persister.results.size() : x.results.size();
        // Output of both operators should be identical
        for (int i = 0; i < maxTuples; i++) {
            logger.debug("Tuple = " + x.results.get(i) + " - " + persister.results.get(i));
            assertEquals("Mismatch observed for tuple ", x.results.get(i), persister.results.get(i));
        }
    } finally {
        x.results.clear();
        persister.results.clear();
    }
}

Example 2 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorMultiplePhysicalOperatorsForSink.

@Test
public void testPersistStreamOperatorMultiplePhysicalOperatorsForSink() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PartitionedTestOperatorWithFiltering passThru = dag.addOperator("partition", new PartitionedTestOperatorWithFiltering());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final TestPersistanceOperator console1 = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input);
    dag.setInputPortAttribute(passThru.input, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.addStream("Stream2", passThru.output, console.inport);
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 100000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if ((console.results.size() < 6) || (console1.results.size() < 6)) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        Integer[] expectedResult = { 0, 1, 4, 5, 8, 9, 12, 13, 16 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(console1.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], console1.results.get(i));
        }
    } finally {
        console1.results.clear();
        console.results.clear();
    }
}

Example 3 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AutoMetricTest method testMetricsAggregations.

@Test
@Ignore
public void testMetricsAggregations() throws Exception {
    CountDownLatch latch = new CountDownLatch(2);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.setOperatorAttribute(o1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(2));
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("progress", 2L, ((Long) aggregator.result.get("progress")).longValue());
    lc.shutdown();
}

Example 4 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AutoMetricTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("progress", 1L, ((Long) aggregator.result.get("progress")).longValue());
    lc.shutdown();
}

Example 5 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AutoMetricTest method testMetricsAnnotatedMethod.

@Test
public void testMetricsAnnotatedMethod() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("myMetric", 3, ((Integer) aggregator.result.get("myMetric")).intValue());
    lc.shutdown();
}