Examples with FileSourceSplit org.apache.flink.connector.file.src.FileSourceSplit used on opensource projects

Example 36 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class StaticFileSplitEnumeratorTest method testCheckpointNoSplitRequested.

@Test
public void testCheckpointNoSplitRequested() throws Exception {
    final TestingSplitEnumeratorContext<FileSourceSplit> context = new TestingSplitEnumeratorContext<>(4);
    final FileSourceSplit split = createRandomSplit();
    final StaticFileSplitEnumerator enumerator = createEnumerator(context, split);
    final PendingSplitsCheckpoint<FileSourceSplit> checkpoint = enumerator.snapshotState(1L);
    assertThat(checkpoint.getSplits(), contains(split));
}

Example 37 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testIncludeFilesFromNestedDirectories.

// ------------------------------------------------------------------------
@Test
public void testIncludeFilesFromNestedDirectories() throws Exception {
    final Path[] testPaths = new Path[] { new Path("testfs:///dir/file1"), new Path("testfs:///dir/nested/file.out"), new Path("testfs:///dir/nested/anotherfile.txt") };
    testFs = TestingFileSystem.createWithFiles("testfs", testPaths);
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///dir") }, 1);
    assertThat(toPaths(splits), containsInAnyOrder(testPaths));
}

Example 38 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testFileWithMultipleBlocks.

@Test
public void testFileWithMultipleBlocks() throws Exception {
    final Path testPath = new Path("testfs:///dir/file");
    testFs = TestingFileSystem.createForFileStatus("testfs", TestingFileSystem.TestFileStatus.forFileWithBlocks(testPath, 1000L, new TestingFileSystem.TestBlockLocation(0L, 100L, "host1", "host2"), new TestingFileSystem.TestBlockLocation(100L, 520L, "host2", "host3"), new TestingFileSystem.TestBlockLocation(620L, 380L, "host3", "host4")));
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///dir") }, 0);
    assertSplitsEqual(new FileSourceSplit("ignoredId", testPath, 0L, 1000L, 0, 1000L, "host1", "host2", "host3", "host4"), splits.iterator().next());
}

Example 39 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testFileWithIncorrectBlocks.

@Test
public void testFileWithIncorrectBlocks() throws Exception {
    final Path testPath = new Path("testfs:///testdir/testfile");
    testFs = TestingFileSystem.createForFileStatus("testfs", TestingFileSystem.TestFileStatus.forFileWithBlocks(testPath, 10000L, new TestingFileSystem.TestBlockLocation(0L, 1000L), new TestingFileSystem.TestBlockLocation(2000L, 1000L)));
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///testdir") }, 0);
    assertEquals(1, splits.size());
    assertSplitsEqual(new FileSourceSplit("ignoredId", testPath, 0L, 10000L, 0, 12345L), splits.iterator().next());
}

Example 40 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testHiddenDirectories.

@Test
public void testHiddenDirectories() throws Exception {
    final Path[] testPaths = new Path[] { new Path("testfs:///dir/visiblefile"), new Path("testfs:///dir/.hiddendir/file"), new Path("testfs:///_notvisible/afile") };
    testFs = TestingFileSystem.createWithFiles("testfs", testPaths);
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///") }, 1);
    assertEquals(Collections.singletonList(new Path("testfs:///dir/visiblefile")), toPaths(splits));
}