Examples with FSWindowDataManager org.apache.apex.malhar.lib.wal.FSWindowDataManager used on opensource projects

Example 11 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class NiFiSinglePortInputOperatorTest method setup.

@Before
public void setup() throws IOException {
    final String windowDataDir = "target/" + this.getClass().getSimpleName();
    final File windowDataDirFile = new File(windowDataDir);
    if (windowDataDirFile.exists()) {
        FileUtils.deleteFile(windowDataDirFile, true);
    }
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);
    context = mockOperatorContext(12345, attributeMap);
    sink = new CollectorTestSink<>();
    builder = new MockSiteToSiteClient.Builder();
    windowDataManager = new FSWindowDataManager();
    operator = new NiFiSinglePortInputOperator(builder, windowDataManager);
    operator.outputPort.setSink(sink);
}

Example 12 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class RabbitMQInputOperatorTest method runTest.

protected void runTest(final int testNum) throws IOException {
    LocalMode lma = LocalMode.newInstance();
    DAG dag = lma.getDAG();
    RabbitMQInputOperator consumer = dag.addOperator("Consumer", RabbitMQInputOperator.class);
    consumer.setWindowDataManager(new FSWindowDataManager());
    final CollectorModule<byte[]> collector = dag.addOperator("Collector", new CollectorModule<byte[]>());
    consumer.setHost("localhost");
    consumer.setExchange("testEx");
    consumer.setExchangeType("fanout");
    final RabbitMQMessageGenerator publisher = new RabbitMQMessageGenerator();
    publisher.setup();
    dag.addStream("Stream", consumer.outputPort, collector.inputPort).setLocality(Locality.CONTAINER_LOCAL);
    final LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    new Thread("LocalClusterController") {

        @Override
        public void run() {
            long startTms = System.currentTimeMillis();
            long timeout = 100000L;
            try {
                while (!collector.inputPort.collections.containsKey("collector") && System.currentTimeMillis() - startTms < timeout) {
                    Thread.sleep(500);
                }
                publisher.generateMessages(testNum);
                startTms = System.currentTimeMillis();
                while (System.currentTimeMillis() - startTms < timeout) {
                    List<?> list = collector.inputPort.collections.get("collector");
                    if (list.size() < testNum * 3) {
                        Thread.sleep(10);
                    } else {
                        break;
                    }
                }
            } catch (IOException ex) {
                logger.error(ex.getMessage(), ex);
                DTThrowable.rethrow(ex);
            } catch (InterruptedException ex) {
                DTThrowable.rethrow(ex);
            } finally {
                lc.shutdown();
            }
        }
    }.start();
    lc.run();
    logger.debug("collection size: {} {}", collector.inputPort.collections.size(), collector.inputPort.collections);
    MessageQueueTestHelper.validateResults(testNum, collector.inputPort.collections);
}

Example 13 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class NiFiSinglePortOutputOperatorTest method setup.

@Before
public void setup() throws IOException {
    final String windowDataDir = "target/" + this.getClass().getSimpleName();
    final File windowDataDirFile = new File(windowDataDir);
    if (windowDataDirFile.exists()) {
        FileUtils.deleteFile(windowDataDirFile, true);
    }
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);
    context = mockOperatorContext(12345, attributeMap);
    windowDataManager = new FSWindowDataManager();
    stsBuilder = new MockSiteToSiteClient.Builder();
    dpBuilder = new StringNiFiDataPacketBuilder();
    operator = new NiFiSinglePortOutputOperator(stsBuilder, dpBuilder, windowDataManager, 1);
}

Example 14 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testIdempotencyWithCheckPoint.

public static <S extends AbstractFileInputOperator, T> void testIdempotencyWithCheckPoint(S oper, CollectorTestSink<T> queryResults, IdempotencyTestDriver<S> driver) throws Exception {
    FileContext.getLocalFSFileContext().delete(new Path(new File(driver.getDirectory()).getAbsolutePath()), true);
    int file = 0;
    driver.writeFile(5, "file" + file);
    file = 1;
    driver.writeFile(6, "file" + file);
    // empty file
    file = 2;
    driver.writeFile(0, "file" + file);
    FSWindowDataManager manager = new FSWindowDataManager();
    manager.setStatePath(driver.getDirectory() + "/recovery");
    oper.setWindowDataManager(manager);
    oper.setDirectory(driver.getDirectory());
    oper.getScanner().setFilePatternRegexp(".*file[\\d]");
    oper.setup(driver.getContext());
    oper.setEmitBatchSize(3);
    // sort the pendingFiles and ensure the ordering of the files scanned
    DirectoryScannerNew newScanner = new DirectoryScannerNew();
    oper.setScanner(newScanner);
    // scan directory
    oper.beginWindow(0);
    oper.emitTuples();
    oper.endWindow();
    // emit f0l0, f0l1, f0l2
    oper.beginWindow(1);
    oper.emitTuples();
    oper.endWindow();
    // checkpoint the operator
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    S checkPointOper = checkpoint(oper, bos);
    // start saving output
    driver.setSink(oper, queryResults);
    // emit f0l3, f0l4, and closeFile(f0) in the same window
    oper.beginWindow(2);
    oper.emitTuples();
    oper.endWindow();
    List<T> beforeRecovery2 = Lists.newArrayList(queryResults.collectedTuples);
    // emit f1l0, f1l1, f1l2
    oper.beginWindow(3);
    oper.emitTuples();
    oper.endWindow();
    List<T> beforeRecovery3 = Lists.newArrayList(queryResults.collectedTuples);
    // emit f1l3, f1l4, f1l5
    oper.beginWindow(4);
    oper.emitTuples();
    oper.endWindow();
    List<T> beforeRecovery4 = Lists.newArrayList(queryResults.collectedTuples);
    // closeFile(f1) in a new window
    oper.beginWindow(5);
    oper.emitTuples();
    oper.endWindow();
    List<T> beforeRecovery5 = Lists.newArrayList(queryResults.collectedTuples);
    // empty file ops, closeFile(f2) in emitTuples() only
    oper.beginWindow(6);
    oper.emitTuples();
    oper.endWindow();
    List<T> beforeRecovery6 = Lists.newArrayList(queryResults.collectedTuples);
    oper.teardown();
    queryResults.clear();
    // idempotency  part
    oper = restoreCheckPoint(checkPointOper, bos);
    driver.getContext().getAttributes().put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1L);
    oper.setup(driver.getContext());
    driver.setSink(oper, queryResults);
    long startwid = driver.getContext().getAttributes().get(Context.OperatorContext.ACTIVATION_WINDOW_ID) + 1;
    oper.beginWindow(startwid);
    Assert.assertTrue(oper.currentFile == null);
    oper.emitTuples();
    oper.endWindow();
    Assert.assertEquals("lines", beforeRecovery2, queryResults.collectedTuples);
    oper.beginWindow(++startwid);
    oper.emitTuples();
    oper.endWindow();
    Assert.assertEquals("lines", beforeRecovery3, queryResults.collectedTuples);
    oper.beginWindow(++startwid);
    oper.emitTuples();
    oper.endWindow();
    Assert.assertEquals("lines", beforeRecovery4, queryResults.collectedTuples);
    oper.beginWindow(++startwid);
    Assert.assertTrue(oper.currentFile == null);
    oper.emitTuples();
    oper.endWindow();
    Assert.assertEquals("lines", beforeRecovery5, queryResults.collectedTuples);
    oper.beginWindow(++startwid);
    Assert.assertTrue(oper.currentFile == null);
    oper.emitTuples();
    oper.endWindow();
    Assert.assertEquals("lines", beforeRecovery6, queryResults.collectedTuples);
    Assert.assertEquals("number tuples", 8, queryResults.collectedTuples.size());
    oper.teardown();
}

Example 15 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testIdempotency.

@Test
public void testIdempotency() throws Exception {
    FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true);
    List<String> allLines = Lists.newArrayList();
    for (int file = 0; file < 2; file++) {
        List<String> lines = Lists.newArrayList();
        for (int line = 0; line < 2; line++) {
            lines.add("f" + file + "l" + line);
        }
        allLines.addAll(lines);
        FileUtils.write(new File(testMeta.dir, "file" + file), StringUtils.join(lines, '\n'));
    }
    LineByLineFileInputOperator oper = new LineByLineFileInputOperator();
    FSWindowDataManager manager = new FSWindowDataManager();
    manager.setStatePath(testMeta.dir + "/recovery");
    oper.setWindowDataManager(manager);
    CollectorTestSink<String> queryResults = new CollectorTestSink<String>();
    TestUtils.setSink(oper.output, queryResults);
    oper.setDirectory(testMeta.dir);
    oper.getScanner().setFilePatternRegexp(".*file[\\d]");
    oper.setup(testMeta.context);
    for (long wid = 0; wid < 3; wid++) {
        oper.beginWindow(wid);
        oper.emitTuples();
        oper.endWindow();
    }
    oper.teardown();
    List<String> beforeRecovery = Lists.newArrayList(queryResults.collectedTuples);
    queryResults.clear();
    // idempotency  part
    oper.setup(testMeta.context);
    for (long wid = 0; wid < 3; wid++) {
        oper.beginWindow(wid);
        oper.endWindow();
    }
    Assert.assertEquals("number tuples", 4, queryResults.collectedTuples.size());
    Assert.assertEquals("lines", beforeRecovery, queryResults.collectedTuples);
    oper.teardown();
}