Examples with StramLocalCluster com.datatorrent.stram.StramLocalCluster used on opensource projects

Example 31 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class ProcessingModeTests method testLinearInputOperatorRecovery.

public void testLinearInputOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    CollectorOperator.duplicates.clear();
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    dag.getMeta(rip).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    if (Operator.ProcessingMode.EXACTLY_ONCE.equals(processingMode)) {
        dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
    }
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
}

Example 32 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class ProcessingModeTests method testLinearOperatorRecovery.

public void testLinearOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    CollectorOperator.duplicates.clear();
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
}

Example 33 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StatsTest method baseTestForQueueSize.

@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    testOper.setMaxTuples(maxTuples);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    if (statsListener != null) {
        dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { statsListener }));
    }
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
    Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
    for (PTOperator p : operatorMap.values()) {
        StramTestSupport.waitForActivation(lc, p);
    }
    long startTms = System.currentTimeMillis();
    if (statsListener != null) {
        while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    } else {
        while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    }
    if (statsListener != null) {
        statsListener.validateStats();
    } else {
        collector.validateStats();
    }
    lc.shutdown();
}

Example 34 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class TupleRecorderTest method testRecordingFlow.

@Test
public void testRecordingFlow() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir.getAbsolutePath(), null));
    dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, "file://" + testWorkDir.getAbsolutePath());
    // 1KB per part
    dag.getAttributes().put(LogicalPlan.TUPLE_RECORDING_PART_FILE_SIZE, 1024);
    TestGeneratorInputOperator op1 = dag.addOperator("op1", TestGeneratorInputOperator.class);
    GenericTestOperator op2 = dag.addOperator("op2", GenericTestOperator.class);
    GenericTestOperator op3 = dag.addOperator("op3", GenericTestOperator.class);
    // emit every 100 msec
    op1.setEmitInterval(100);
    //.setInline(true);
    dag.addStream("stream1", op1.outport, op2.inport1);
    //.setInline(true);
    dag.addStream("stream2", op2.outport1, op3.inport1);
    final StramLocalCluster localCluster = new StramLocalCluster(dag);
    localCluster.runAsync();
    final PTOperator ptOp2 = localCluster.findByLogicalNode(dag.getMeta(op2));
    StramTestSupport.waitForActivation(localCluster, ptOp2);
    testRecordingOnOperator(localCluster, ptOp2);
    final PTOperator ptOp1 = localCluster.findByLogicalNode(dag.getMeta(op1));
    StramTestSupport.waitForActivation(localCluster, ptOp1);
    testRecordingOnOperator(localCluster, ptOp1);
    localCluster.shutdown();
}

Example 35 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class AtLeastOnceTest method testOperatorRecovery.

@Test
public void testOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}