Examples with StramLocalCluster com.datatorrent.stram.StramLocalCluster used on opensource projects

Example 41 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOnSingleSinkWithFilteringContainerLocal.

@Test
public void testPersistStreamOnSingleSinkWithFilteringContainerLocal() throws ClassNotFoundException, IOException, InterruptedException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PassThruOperatorWithCodec passThru = dag.addOperator("PassThrough", new PassThruOperatorWithCodec(2));
    PassThruOperatorWithCodec passThru2 = dag.addOperator("Multiples_of_3", new PassThruOperatorWithCodec(3));
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final TestReceiverOperator console1 = dag.addOperator("console1", new TestReceiverOperator());
    TestPersistanceOperator persister = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input, passThru2.input).setLocality(Locality.CONTAINER_LOCAL);
    s.persistUsing("persister", persister, persister.inport);
    dag.addStream("Stream2", passThru.output, console.inport);
    dag.addStream("Stream3", passThru2.output, console1.inport);
    // runLocalClusterAndValidate(dag, console, persister);
    persister.results.clear();
    console.results.clear();
    console1.results.clear();
    // Validate union of results is received on persist operator
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 1000000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if ((console.results.size() < 6) || (console.results.size() < 6)) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        Integer[] expectedResult = { 0, 2, 3, 4, 6, 8, 9, 10, 12 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(persister.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], persister.results.get(i));
        }
    } finally {
        persister.results.clear();
        console.results.clear();
        console1.results.clear();
    }
}

Example 42 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorGeneratesUnionOfAllSinksOutput.

@Test
public void testPersistStreamOperatorGeneratesUnionOfAllSinksOutput() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PassThruOperatorWithCodec passThru1 = dag.addOperator("PassThrough1", new PassThruOperatorWithCodec(2));
    PassThruOperatorWithCodec passThru2 = dag.addOperator("PassThrough2", new PassThruOperatorWithCodec(3));
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    final TestReceiverOperator console1 = dag.addOperator("console1", new TestReceiverOperator());
    TestPersistanceOperator persister = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru1.input, passThru2.input);
    s.persistUsing("persister", persister, persister.inport);
    dag.addStream("Stream2", passThru1.output, console.inport);
    dag.addStream("Stream3", passThru2.output, console1.inport);
    persister.results.clear();
    console.results.clear();
    console1.results.clear();
    // Validate union of results is received on persist operator
    final StramLocalCluster lc = new StramLocalCluster(dag);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 10000L;
            try {
                while (System.currentTimeMillis() - startTms < timeout) {
                    if ((console.results.size() < 6) || (console.results.size() < 6)) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (Exception ex) {
                throw Throwables.propagate(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    try {
        Integer[] expectedResult = { 0, 2, 3, 4, 6, 8, 9, 10, 12 };
        for (int i = 0; i < expectedResult.length; i++) {
            logger.debug(persister.results.get(i) + " " + expectedResult[i]);
            assertEquals("Mismatch observed for tuple ", expectedResult[i], persister.results.get(i));
        }
    } finally {
        persister.results.clear();
        console.results.clear();
        console1.results.clear();
    }
}

Example 43 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class OiOEndWindowTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
    FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
    SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());
    /*
     * This tests make sure that even if the application_window_count is different the endWindow() is called for
     * end_stream
     */
    lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);
    stream1.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}

Example 44 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class OiOStreamTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    ThreadIdValidatingInputOperator io = lp.addOperator("Input Operator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingOutputOperator go = lp.addOperator("Output Operator", new ThreadIdValidatingOutputOperator());
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    /* The first test makes sure that when they are not ThreadLocal they use different threads */
    ThreadIdValidatingOutputOperator.threadList.clear();
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertFalse("Thread Id", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingOutputOperator.threadId);
    /* This test makes sure that since they are ThreadLocal, they indeed share a thread */
    ThreadIdValidatingOutputOperator.threadList.clear();
    stream.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("Thread Id", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingOutputOperator.threadId);
}

Example 45 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-malhar by apache.

the class MinimalWordCountTest method MinimalWordCountTest.

@Test
public void MinimalWordCountTest() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.set("dt.application.MinimalWordCount.operator.console.silent", "true");
    MinimalWordCount app = new MinimalWordCount();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return MinimalWordCount.Collector.isDone();
        }
    });
    lc.run(10000);
    Assert.assertTrue(MinimalWordCount.Collector.result.get("error") == 7);
    Assert.assertTrue(MinimalWordCount.Collector.result.get("word") == 119);
    Assert.assertTrue(MinimalWordCount.Collector.result.get("bye") == 1);
}