Examples with FileSourceSplit org.apache.flink.connector.file.src.FileSourceSplit used on opensource projects

Example 16 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class LocalityAwareSplitAssignerTest method testAssignmentMultiLocalHost.

@SuppressWarnings("UnnecessaryLocalVariable")
@Test
public void testAssignmentMultiLocalHost() {
    final String[] localHosts = { "local1", "local2", "local3" };
    final String[] remoteHosts = { "remote1", "remote2", "remote3" };
    final String[] requestingHosts = { "local3", "local2", "local1", "other" };
    final int numThreeLocalSplits = 10;
    final int numTwoLocalSplits = 10;
    final int numOneLocalSplits = 10;
    final int numLocalSplits = 30;
    final int numRemoteSplits = 10;
    final int numSplits = 40;
    final String[] threeLocalHosts = localHosts;
    final String[] twoLocalHosts = { localHosts[0], localHosts[1], remoteHosts[0] };
    final String[] oneLocalHost = { localHosts[0], remoteHosts[0], remoteHosts[1] };
    final String[] noLocalHost = remoteHosts;
    int splitCnt = 0;
    final Set<FileSourceSplit> splits = new HashSet<>();
    // add splits with three local hosts
    for (int i = 0; i < numThreeLocalSplits; i++) {
        splits.add(createSplit(splitCnt++, threeLocalHosts));
    }
    // add splits with two local hosts
    for (int i = 0; i < numTwoLocalSplits; i++) {
        splits.add(createSplit(splitCnt++, twoLocalHosts));
    }
    // add splits with two local hosts
    for (int i = 0; i < numOneLocalSplits; i++) {
        splits.add(createSplit(splitCnt++, oneLocalHost));
    }
    // add splits with two local hosts
    for (int i = 0; i < numRemoteSplits; i++) {
        splits.add(createSplit(splitCnt++, noLocalHost));
    }
    // get all available splits
    final LocalityAwareSplitAssigner ia = new LocalityAwareSplitAssigner(splits);
    for (int i = 0; i < numSplits; i++) {
        final String host = requestingHosts[i % requestingHosts.length];
        final Optional<FileSourceSplit> ois = ia.getNext(host);
        assertTrue(ois.isPresent());
        final FileSourceSplit is = ois.get();
        assertTrue(splits.remove(is));
        // check priority of split
        if (host.equals(localHosts[0])) {
            assertArrayEquals(is.hostnames(), oneLocalHost);
        } else if (host.equals(localHosts[1])) {
            assertArrayEquals(is.hostnames(), twoLocalHosts);
        } else if (host.equals(localHosts[2])) {
            assertArrayEquals(is.hostnames(), threeLocalHosts);
        } else {
            assertArrayEquals(is.hostnames(), noLocalHost);
        }
    }
    // check we had all
    assertTrue(splits.isEmpty());
    assertFalse(ia.getNext("anotherHost").isPresent());
    assertEquals(numRemoteSplits, ia.getNumberOfRemoteAssignments());
    assertEquals(numLocalSplits, ia.getNumberOfLocalAssignments());
}

Example 17 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testFilesWithNoBlockInfo.

@Test
public void testFilesWithNoBlockInfo() throws Exception {
    final Path testPath = new Path("testfs:///dir/file1");
    testFs = TestingFileSystem.createForFileStatus("testfs", TestingFileSystem.TestFileStatus.forFileWithBlocks(testPath, 12345L));
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///dir") }, 0);
    assertEquals(1, splits.size());
    assertSplitsEqual(new FileSourceSplit("ignoredId", testPath, 0L, 12345L, 0, 12345L), splits.iterator().next());
}

Example 18 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class NonSplittingRecursiveEnumeratorTest method testDefaultHiddenFilesFilter.

@Test
public void testDefaultHiddenFilesFilter() throws Exception {
    final Path[] testPaths = new Path[] { new Path("testfs:///visiblefile"), new Path("testfs:///.hiddenfile1"), new Path("testfs:///_hiddenfile2") };
    testFs = TestingFileSystem.createWithFiles("testfs", testPaths);
    testFs.register();
    final NonSplittingRecursiveEnumerator enumerator = createEnumerator();
    final Collection<FileSourceSplit> splits = enumerator.enumerateSplits(new Path[] { new Path("testfs:///") }, 1);
    assertEquals(Collections.singletonList(new Path("testfs:///visiblefile")), toPaths(splits));
}

Example 19 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class ContinuousFileSplitEnumeratorTest method testDiscoverSplitWhenNoReaderRegistered.

@Test
public void testDiscoverSplitWhenNoReaderRegistered() throws Exception {
    final TestingFileEnumerator fileEnumerator = new TestingFileEnumerator();
    final TestingSplitEnumeratorContext<FileSourceSplit> context = new TestingSplitEnumeratorContext<>(4);
    final ContinuousFileSplitEnumerator enumerator = createEnumerator(fileEnumerator, context);
    // make one split available and trigger the periodic discovery
    final FileSourceSplit split = createRandomSplit();
    fileEnumerator.addSplits(split);
    context.triggerAllActions();
    assertThat(enumerator.snapshotState(1L).getSplits(), contains(split));
}

Example 20 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class ContinuousFileSplitEnumeratorTest method testDiscoverWhenReaderRegistered.

@Test
public void testDiscoverWhenReaderRegistered() throws Exception {
    final TestingFileEnumerator fileEnumerator = new TestingFileEnumerator();
    final TestingSplitEnumeratorContext<FileSourceSplit> context = new TestingSplitEnumeratorContext<>(4);
    final ContinuousFileSplitEnumerator enumerator = createEnumerator(fileEnumerator, context);
    // register one reader, and let it request a split
    context.registerReader(2, "localhost");
    enumerator.addReader(2);
    enumerator.handleSplitRequest(2, "localhost");
    // make one split available and trigger the periodic discovery
    final FileSourceSplit split = createRandomSplit();
    fileEnumerator.addSplits(split);
    context.triggerAllActions();
    assertThat(enumerator.snapshotState(1L).getSplits(), empty());
    assertThat(context.getSplitAssignments().get(2).getAssignedSplits(), contains(split));
}