Examples with StramLocalCluster com.datatorrent.stram.StramLocalCluster used on opensource projects

Example 26 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StreamPersistanceTests method testPartitionedPersistOperator.

@Test
public void testPartitionedPersistOperator() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PartitionedTestOperatorWithFiltering passThru = dag.addOperator("partition", new PartitionedTestOperatorWithFiltering());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input);
    dag.setInputPortAttribute(passThru.input, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.setInputPortAttribute(console1.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
    dag.addStream("Stream2", passThru.output, console.inport);
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 100000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if (console1.results.size() < 6) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        // Values as per persist operator's partition keys should be picked up
        Integer[] expectedResult = { 0, 4, 8, 12, 16, 20 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(console1.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], console1.results.get(i));
        }
    } finally {
        console1.results.clear();
        console.results.clear();
    }
}

Example 27 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class NodeTest method testStreamingWindowGenericNode.

@Ignore
@Test
public void testStreamingWindowGenericNode() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 10);
    dag.addOperator("GenericOperator", new TestGenericOperator());
    final StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run(2000);
}

Example 28 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AtLeastOnceTest method testInputOperatorRecovery.

@Test
public void testInputOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testInputOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 29 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AutoMetricTest method testMetricPropagation.

/**
   * Verify custom stats generated by operator are propagated and trigger repartition.
   *
   * @throws Exception
   */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testMetricPropagation() throws Exception {
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestStatsListener sl = new TestStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) sl));
    GenericTestOperator collector = dag.addOperator("Collector", new GenericTestOperator());
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    long startTms = System.currentTimeMillis();
    while (TestOperator.lastMetric == null && StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        Thread.sleep(300);
        LOG.debug("Waiting for stats");
    }
    while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        if (sl.lastPropVal) {
            break;
        }
        Thread.sleep(100);
        LOG.debug("Waiting for property set");
    }
    lc.shutdown();
    Assert.assertNotNull("metric received", TestOperator.lastMetric);
    Assert.assertEquals("metric message", "interesting", TestOperator.lastMetric.message);
    Assert.assertTrue("attribute defined stats listener called", TestOperator.lastMetric.attributeListenerCalled);
    Assert.assertSame("single thread", TestOperator.definePartitionsThread, TestOperator.processStatsThread);
    Assert.assertTrue("property set", sl.lastPropVal);
}

Example 30 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AutoMetricTest method testDefaultMetricsAggregator.

@Test
public void testDefaultMetricsAggregator() throws Exception {
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    CountDownLatch latch = new CountDownLatch(1);
    OperatorAndAggregator o1 = dag.addOperator("o1", new OperatorAndAggregator(latch));
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
    Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("progress", 1, o1.result.get("progress"));
    lc.shutdown();
}