Examples with FSWindowDataManager org.apache.apex.malhar.lib.wal.FSWindowDataManager used on opensource projects

Example 1 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class RabbitMQInputOperatorTest method testRecoveryAndIdempotency.

@Test
public void testRecoveryAndIdempotency() throws Exception {
    RabbitMQInputOperator operator = new RabbitMQInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setHost("localhost");
    operator.setExchange("testEx");
    operator.setExchangeType("fanout");
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    operator.outputPort.setSink(sink);
    OperatorContext context = mockOperatorContext(1, attributeMap);
    operator.setup(context);
    operator.activate(context);
    final RabbitMQMessageGenerator publisher = new RabbitMQMessageGenerator();
    publisher.setup();
    publisher.generateMessages(5);
    Thread.sleep(10000);
    operator.beginWindow(1);
    operator.emitTuples();
    operator.endWindow();
    operator.deactivate();
    Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());
    // failure and then re-deployment of operator
    sink.collectedTuples.clear();
    operator.setup(context);
    operator.activate(context);
    Assert.assertEquals("largest recovery window", 1, operator.getWindowDataManager().getLargestCompletedWindow());
    operator.beginWindow(1);
    operator.endWindow();
    Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());
    sink.collectedTuples.clear();
    operator.deactivate();
    operator.teardown();
    operator.getWindowDataManager().committed(1);
    publisher.teardown();
}

Example 2 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class RabbitMQOutputOperatorTest method runTest.

protected void runTest(int testNum) throws IOException {
    RabbitMQMessageReceiver receiver = new RabbitMQMessageReceiver();
    receiver.setup();
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    SourceModule source = dag.addOperator("source", new SourceModule());
    source.setTestNum(testNum);
    RabbitMQOutputOperator collector = dag.addOperator("generator", new RabbitMQOutputOperator());
    collector.setWindowDataManager(new FSWindowDataManager());
    collector.setExchange("testEx");
    dag.addStream("Stream", source.outPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    try {
        Thread.sleep(1000);
        long timeout = 10000L;
        long startTms = System.currentTimeMillis();
        while ((receiver.count < testNum * 3) && (System.currentTimeMillis() - startTms < timeout)) {
            Thread.sleep(100);
        }
    } catch (InterruptedException ex) {
        Assert.fail(ex.getMessage());
    } finally {
        lc.shutdown();
    }
    Assert.assertEquals("emitted value for testNum was ", testNum * 3, receiver.count);
    for (Map.Entry<String, Integer> e : receiver.dataMap.entrySet()) {
        if (e.getKey().equals("a")) {
            Assert.assertEquals("emitted value for 'a' was ", new Integer(2), e.getValue());
        } else if (e.getKey().equals("b")) {
            Assert.assertEquals("emitted value for 'b' was ", new Integer(20), e.getValue());
        } else if (e.getKey().equals("c")) {
            Assert.assertEquals("emitted value for 'c' was ", new Integer(1000), e.getValue());
        }
    }
}

Example 3 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class ExactlyOnceFileOutputApp method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    KafkaSinglePortStringInputOperator kafkaInput = dag.addOperator("kafkaInput", new KafkaSinglePortStringInputOperator());
    kafkaInput.setWindowDataManager(new FSWindowDataManager());
    ExactlyOnceJdbcOutputApp.UniqueCounterFlat count = dag.addOperator("count", new ExactlyOnceJdbcOutputApp.UniqueCounterFlat());
    FileWriter fileWriter = dag.addOperator("fileWriter", new FileWriter());
    ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("words", kafkaInput.outputPort, count.data);
    dag.addStream("counts", count.counts, fileWriter.input, cons.input);
}

Example 4 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class RedisInputOperatorTest method testRecoveryAndIdempotency.

@Test
public void testRecoveryAndIdempotency() throws Exception {
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(1);
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    RedisKeyValueInputOperator operator = new RedisKeyValueInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setStore(operatorStore);
    operator.setScanCount(1);
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    operator.outputPort.setSink(sink);
    OperatorContext context = mockOperatorContext(1, attributeMap);
    try {
        operator.setup(context);
        operator.beginWindow(1);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow1 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow2 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        // failure and then re-deployment of operator
        // Re-instantiating to reset values
        operator = new RedisKeyValueInputOperator();
        operator.setWindowDataManager(new FSWindowDataManager());
        operator.setStore(operatorStore);
        operator.setScanCount(1);
        operator.outputPort.setSink(sink);
        operator.setup(context);
        Assert.assertEquals("largest recovery window", 2, operator.getWindowDataManager().getLargestCompletedWindow());
        operator.beginWindow(1);
        operator.emitTuples();
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 1", numberOfMessagesInWindow1, sink.collectedTuples.size());
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 2", numberOfMessagesInWindow2, sink.collectedTuples.size());
    } finally {
        for (Object e : sink.collectedTuples) {
            KeyValPair<String, String> entry = (KeyValPair<String, String>) e;
            testStore.remove(entry.getKey());
        }
        sink.collectedTuples.clear();
        operator.getWindowDataManager().committed(5);
        operator.teardown();
    }
}

Example 5 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testIdempotencyWhenFileContinued.

@Test
public void testIdempotencyWhenFileContinued() throws Exception {
    FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true);
    List<String> lines = Lists.newArrayList();
    for (int line = 0; line < 10; line++) {
        lines.add("l" + line);
    }
    FileUtils.write(new File(testMeta.dir, "file0"), StringUtils.join(lines, '\n'));
    LineByLineFileInputOperator oper = new LineByLineFileInputOperator();
    FSWindowDataManager manager = new FSWindowDataManager();
    manager.setStatePath(testMeta.dir + "/recovery");
    oper.setEmitBatchSize(5);
    oper.setWindowDataManager(manager);
    CollectorTestSink<String> queryResults = new CollectorTestSink<String>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> sink = (CollectorTestSink) queryResults;
    oper.output.setSink(sink);
    oper.setDirectory(testMeta.dir);
    oper.getScanner().setFilePatternRegexp(".*file[\\d]");
    oper.setup(testMeta.context);
    int offset = 0;
    for (long wid = 0; wid < 3; wid++) {
        oper.beginWindow(wid);
        oper.emitTuples();
        oper.endWindow();
        if (wid > 0) {
            Assert.assertEquals("number tuples", 5, queryResults.collectedTuples.size());
            Assert.assertEquals("lines", lines.subList(offset, offset + 5), queryResults.collectedTuples);
            offset += 5;
        }
        sink.clear();
    }
    oper.teardown();
    sink.clear();
    // idempotency  part
    offset = 0;
    oper.setup(testMeta.context);
    for (long wid = 0; wid < 3; wid++) {
        oper.beginWindow(wid);
        oper.endWindow();
        if (wid > 0) {
            Assert.assertEquals("number tuples", 5, queryResults.collectedTuples.size());
            Assert.assertEquals("lines", lines.subList(offset, offset + 5), queryResults.collectedTuples);
            offset += 5;
        }
        sink.clear();
    }
    oper.teardown();
}