Examples with AsyncFSStorageAgent com.datatorrent.common.util.AsyncFSStorageAgent used on opensource projects

Example 11 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class StatsTest method baseTestForQueueSize.

@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    testOper.setMaxTuples(maxTuples);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    if (statsListener != null) {
        dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { statsListener }));
    }
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
    Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
    for (PTOperator p : operatorMap.values()) {
        StramTestSupport.waitForActivation(lc, p);
    }
    long startTms = System.currentTimeMillis();
    if (statsListener != null) {
        while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    } else {
        while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    }
    if (statsListener != null) {
        statsListener.validateStats();
    } else {
        collector.validateStats();
    }
    lc.shutdown();
}

Example 12 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class TupleRecorderTest method testRecordingFlow.

@Test
public void testRecordingFlow() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir.getAbsolutePath(), null));
    dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, "file://" + testWorkDir.getAbsolutePath());
    // 1KB per part
    dag.getAttributes().put(LogicalPlan.TUPLE_RECORDING_PART_FILE_SIZE, 1024);
    TestGeneratorInputOperator op1 = dag.addOperator("op1", TestGeneratorInputOperator.class);
    GenericTestOperator op2 = dag.addOperator("op2", GenericTestOperator.class);
    GenericTestOperator op3 = dag.addOperator("op3", GenericTestOperator.class);
    // emit every 100 msec
    op1.setEmitInterval(100);
    //.setInline(true);
    dag.addStream("stream1", op1.outport, op2.inport1);
    //.setInline(true);
    dag.addStream("stream2", op2.outport1, op3.inport1);
    final StramLocalCluster localCluster = new StramLocalCluster(dag);
    localCluster.runAsync();
    final PTOperator ptOp2 = localCluster.findByLogicalNode(dag.getMeta(op2));
    StramTestSupport.waitForActivation(localCluster, ptOp2);
    testRecordingOnOperator(localCluster, ptOp2);
    final PTOperator ptOp1 = localCluster.findByLogicalNode(dag.getMeta(op1));
    StramTestSupport.waitForActivation(localCluster, ptOp1);
    testRecordingOnOperator(localCluster, ptOp1);
    localCluster.shutdown();
}

Example 13 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class AtLeastOnceTest method testOperatorRecovery.

@Test
public void testOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 14 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class AtLeastOnceTest method testInputOperatorRecovery.

@Test
public void testInputOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testInputOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 15 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class OiOEndWindowTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
    FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
    SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());
    /*
     * This tests make sure that even if the application_window_count is different the endWindow() is called for
     * end_stream
     */
    lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);
    stream1.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}