Examples with FSWindowDataManager org.apache.apex.malhar.lib.wal.FSWindowDataManager used on opensource projects

Example 21 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterInputTest method testFirstWindowAfterRecovery.

@Test
public void testFirstWindowAfterRecovery() throws IOException, InterruptedException {
    FSWindowDataManager fsWindowDataManager = new FSWindowDataManager();
    testMeta.updateConfig(fsWindowDataManager, 500, 2L, 10);
    testMeta.fileSplitterInput.setup(testMeta.context);
    validateRecovery(8, 2);
    Thread.sleep(1000);
    HashSet<String> lines = Sets.newHashSet();
    for (int line = 2; line < 4; line++) {
        lines.add("f13" + "l" + line);
    }
    File f13 = new File(testMeta.dataDirectory, "file13" + ".txt");
    FileUtils.writeLines(f13, lines, true);
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitterInput.beginWindow(9);
    ((MockScanner) testMeta.fileSplitterInput.getScanner()).semaphore.acquire();
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    Assert.assertEquals("Files " + testMeta.fileMetadataSink.collectedTuples, 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 6, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileSplitterInput.teardown();
}

Example 22 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterInputTest method testRecoveryOfPartialFile.

@Test
public void testRecoveryOfPartialFile() throws InterruptedException {
    FSWindowDataManager fsIdempotentStorageManager = new FSWindowDataManager();
    testMeta.updateConfig(fsIdempotentStorageManager, 500L, 2L, 2);
    FileSplitterInput checkpointedInput = KryoCloneUtils.cloneObject(testMeta.fileSplitterInput);
    testMeta.fileSplitterInput.setup(testMeta.context);
    testMeta.fileSplitterInput.beginWindow(1);
    testMeta.scanner.semaphore.acquire(12);
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    // fileX.txt has just 6 blocks. Since blocks threshold is 2, only 2 are emitted.
    Assert.assertEquals("Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    AbstractFileSplitter.FileMetadata fileX = testMeta.fileMetadataSink.collectedTuples.get(0);
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitterInput.teardown();
    // there was a failure and the operator was re-deployed
    testMeta.fileSplitterInput = checkpointedInput;
    testMeta.resetSinks();
    testMeta.fileSplitterInput.setup(testMeta.context);
    testMeta.fileSplitterInput.beginWindow(1);
    // fileX is recovered and first two blocks are repeated.
    Assert.assertEquals("Recovered Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    AbstractFileSplitter.FileMetadata fileXRecovered = testMeta.fileMetadataSink.collectedTuples.get(0);
    Assert.assertEquals("recovered file-metadata", fileX.getFileName(), fileXRecovered.getFileName());
    Assert.assertEquals("Recovered Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileSplitterInput.endWindow();
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitterInput.beginWindow(2);
    // next 2 blocks of fileX
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    testMeta.fileSplitterInput.beginWindow(3);
    // next 2 blocks of fileX
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    // Next 2 blocks of fileX
    Assert.assertEquals("File", 0, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 4, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitterInput.beginWindow(4);
    ((MockScanner) testMeta.fileSplitterInput.getScanner()).semaphore.acquire(11);
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    // 2 blocks of a different file
    Assert.assertEquals("New file", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    AbstractFileSplitter.FileMetadata fileY = testMeta.fileMetadataSink.collectedTuples.get(0);
    for (BlockMetadata.FileBlockMetadata blockMetadata : testMeta.blockMetadataSink.collectedTuples) {
        Assert.assertTrue("Block file name", blockMetadata.getFilePath().endsWith(fileY.getFileName()));
        testMeta.fileSplitterInput.teardown();
    }
}

Example 23 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterInputTest method testRecoveryOfBlockMetadataIterator.

@Test
public void testRecoveryOfBlockMetadataIterator() throws InterruptedException {
    FSWindowDataManager fsWindowDataManager = new FSWindowDataManager();
    testMeta.updateConfig(fsWindowDataManager, 500L, 2L, 2);
    testMeta.fileSplitterInput.setup(testMeta.context);
    testMeta.fileSplitterInput.beginWindow(1);
    testMeta.scanner.semaphore.acquire(12);
    testMeta.fileSplitterInput.emitTuples();
    testMeta.fileSplitterInput.endWindow();
    // file0.txt has just 5 blocks. Since blocks threshold is 2, only 2 are emitted.
    Assert.assertEquals("Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    // At this point the operator was check-pointed and then there was a failure.
    testMeta.fileSplitterInput.teardown();
    // The operator was restored from persisted state and re-deployed.
    testMeta.fileSplitterInput = KryoCloneUtils.cloneObject(testMeta.fileSplitterInput);
    TestUtils.setSink(testMeta.fileSplitterInput.blocksMetadataOutput, testMeta.blockMetadataSink);
    TestUtils.setSink(testMeta.fileSplitterInput.filesMetadataOutput, testMeta.fileMetadataSink);
    testMeta.fileSplitterInput.setup(testMeta.context);
    testMeta.fileSplitterInput.beginWindow(1);
    Assert.assertEquals("Recovered Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Recovered Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileSplitterInput.teardown();
}

Example 24 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterTest method testIdempotency.

@Test
public void testIdempotency() throws InterruptedException {
    FSWindowDataManager fsWindowDataManager = new FSWindowDataManager();
    testMeta.fileSplitter.setWindowDataManager(fsWindowDataManager);
    testMeta.fileSplitter.setup(testMeta.context);
    // will emit window 1 from data directory
    testFileMetadata();
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitter.setup(testMeta.context);
    testMeta.fileSplitter.beginWindow(1);
    Assert.assertEquals("Blocks", 12, testMeta.blockMetadataSink.collectedTuples.size());
    for (Object blockMetadata : testMeta.blockMetadataSink.collectedTuples) {
        BlockMetadata.FileBlockMetadata metadata = (BlockMetadata.FileBlockMetadata) blockMetadata;
        Assert.assertTrue("path: " + metadata.getFilePath(), testMeta.filePaths.contains(metadata.getFilePath()));
    }
}

Example 25 with FSWindowDataManager

use of org.apache.apex.malhar.lib.wal.FSWindowDataManager in project apex-malhar by apache.

the class FileSplitterTest method testRecoveryOfPartialFile.

@Ignore
public void testRecoveryOfPartialFile() throws InterruptedException {
    FSWindowDataManager fsIdempotentStorageManager = new FSWindowDataManager();
    fsIdempotentStorageManager.setStatePath(testMeta.dataDirectory + '/' + "recovery");
    testMeta.fileSplitter.setWindowDataManager(fsIdempotentStorageManager);
    testMeta.fileSplitter.setBlockSize(2L);
    testMeta.fileSplitter.setBlocksThreshold(2);
    testMeta.fileSplitter.scanner.setScanIntervalMillis(500);
    testMeta.fileSplitter.setup(testMeta.context);
    testMeta.fileSplitter.beginWindow(1);
    testMeta.exchanger.exchange(null);
    testMeta.fileSplitter.emitTuples();
    testMeta.fileSplitter.endWindow();
    // file0.txt has just 5 blocks. Since blocks threshold is 2, only 2 are emitted.
    Assert.assertEquals("Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    // there was a failure and the operator was re-deployed
    testMeta.fileSplitter.setup(testMeta.context);
    testMeta.fileSplitter.beginWindow(1);
    Assert.assertEquals("Recovered Files", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Recovered Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    testMeta.fileSplitter.beginWindow(2);
    testMeta.fileSplitter.emitTuples();
    testMeta.fileSplitter.endWindow();
    Assert.assertEquals("Blocks", 4, testMeta.blockMetadataSink.collectedTuples.size());
    String file1 = testMeta.fileMetadataSink.collectedTuples.get(0).getFileName();
    testMeta.fileMetadataSink.clear();
    testMeta.blockMetadataSink.clear();
    testMeta.fileSplitter.beginWindow(3);
    testMeta.exchanger.exchange(null);
    testMeta.fileSplitter.emitTuples();
    testMeta.fileSplitter.endWindow();
    Assert.assertEquals("New file", 1, testMeta.fileMetadataSink.collectedTuples.size());
    Assert.assertEquals("Blocks", 2, testMeta.blockMetadataSink.collectedTuples.size());
    String file2 = testMeta.fileMetadataSink.collectedTuples.get(0).getFileName();
    Assert.assertTrue("Block file name 0", testMeta.blockMetadataSink.collectedTuples.get(0).getFilePath().endsWith(file1));
    Assert.assertTrue("Block file name 1", testMeta.blockMetadataSink.collectedTuples.get(1).getFilePath().endsWith(file2));
}