Examples with AsyncFSStorageAgent com.datatorrent.common.util.AsyncFSStorageAgent used on opensource projects

Example 1 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class AtLeastOnceTest method testInlineOperatorsRecovery.

@Test
public void testInlineOperatorsRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    //dag.getAttributes().get(DAG.HEARTBEAT_INTERVAL_MILLIS, 400);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    //    for (Long l: collection) {
    //      logger.debug(Codec.getStringWindowId(l));
    //    }
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 2 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class InputOperatorTest method testSomeMethod.

@Test
public void testSomeMethod() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String testWorkDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir, null));
    EvenOddIntegerGeneratorInputOperator generator = dag.addOperator("NumberGenerator", EvenOddIntegerGeneratorInputOperator.class);
    final CollectorModule<Number> collector = dag.addOperator("NumberCollector", new CollectorModule<Number>());
    dag.addStream("EvenIntegers", generator.even, collector.even).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("OddIntegers", generator.odd, collector.odd).setLocality(Locality.CONTAINER_LOCAL);
    final StramLocalCluster lc = new StramLocalCluster(dag);
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    WaitCondition c = new WaitCondition() {

        @Override
        public boolean isComplete() {
            return tupleCount.get() > 2;
        }
    };
    StramTestSupport.awaitCompletion(c, 2000);
    lc.shutdown();
    Assert.assertEquals("Collections size", 2, collections.size());
    Assert.assertFalse("Zero tuple count", collections.get(collector.even.id).isEmpty() && collections.get(collector.odd.id).isEmpty());
    Assert.assertTrue("Tuple count", collections.get(collector.even.id).size() - collections.get(collector.odd.id).size() <= 1);
}

Example 3 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class SliderTest method test.

private void test(int applicationWindowCount, int slideByWindowCount) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/sliderTest").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 100);
    Input input = dag.addOperator("Input", new Input());
    Sum sum = dag.addOperator("Sum", new Sum());
    dag.setOperatorAttribute(sum, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindowCount);
    dag.setOperatorAttribute(sum, OperatorContext.SLIDE_BY_WINDOW_COUNT, slideByWindowCount);
    Validator validate = dag.addOperator("validator", new Validator());
    Validator.numbersValidated = 0;
    validate.numberOfIntegers = applicationWindowCount;
    validate.slideByNumbers = slideByWindowCount;
    dag.addStream("input-sum", input.defaultOutputPort, sum.inputPort);
    dag.addStream("sum-validator", sum.outputPort, validate.validate);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    long startTms = System.currentTimeMillis();
    while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        if (validate.numbersValidated > 5) {
            break;
        }
        Thread.sleep(100);
    }
    lc.shutdown();
    Assert.assertTrue("numbers validated more than zero ", validate.numbersValidated > 0);
}

Example 4 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class StatsTest method testPortStatsPropagation.

/**
   * Verify buffer server bytes and tuple count.
   *
   * @throws Exception
   */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testPortStatsPropagation() throws Exception {
    int tupleCount = 10;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestInputStatsListener testInputStatsListener = new TestInputStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testInputStatsListener }));
    testOper.setMaxTuples(tupleCount);
    testOper.setEmitInterval(0);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    TestCollectorStatsListener testCollectorStatsListener = new TestCollectorStatsListener();
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testCollectorStatsListener }));
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(null);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertFalse("input operator stats", testInputStatsListener.inputOperatorStats.isEmpty());
    Assert.assertFalse("collector operator stats", testCollectorStatsListener.collectorOperatorStats.isEmpty());
    try {
        int outputPortTupleCount = 0;
        long outputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testInputStatsListener.inputOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats outputPortStats : operatorStats.outputPorts) {
                outputPortTupleCount += outputPortStats.tupleCount;
                outputPortBufferServerBytes += outputPortStats.bufferServerBytes;
            }
        }
        int inputPortTupleCount = 0;
        long inputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testCollectorStatsListener.collectorOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats inputPortStats : operatorStats.inputPorts) {
                inputPortTupleCount += inputPortStats.tupleCount;
                inputPortBufferServerBytes += inputPortStats.bufferServerBytes;
            }
        }
        Assert.assertEquals("Tuple Count emitted", tupleCount, outputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", inputPortBufferServerBytes > 0);
        Assert.assertEquals("Tuple Count processed", tupleCount, inputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", outputPortBufferServerBytes > 0);
    } finally {
        lc.shutdown();
    }
}

Example 5 with AsyncFSStorageAgent

use of com.datatorrent.common.util.AsyncFSStorageAgent in project apex-core by apache.

the class StreamingContainerTest method testOiOCommitted.

@Test
public void testOiOCommitted() throws IOException, ClassNotFoundException {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/testCommitted").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
    String op1Name = "CommitAwareOperatorTestOioCommit1";
    String op2Name = "CommitAwareOperatorTestOioCommit2";
    CommitAwareOperator operator1 = lp.addOperator(op1Name, new CommitAwareOperator());
    CommitAwareOperator operator2 = lp.addOperator(op2Name, new CommitAwareOperator());
    lp.addStream("local", operator1.output, operator2.input).setLocality(Locality.THREAD_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(lp);
    lc.run(5000);
    /* this is not foolproof but some insurance is better than nothing */
    Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op1Name));
    Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op2Name));
}