Examples with FileSourceSplit org.apache.flink.connector.file.src.FileSourceSplit used on opensource projects

Example 21 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class ContinuousFileSplitEnumeratorTest method testRequestingReaderUnavailableWhenSplitDiscovered.

@Test
public void testRequestingReaderUnavailableWhenSplitDiscovered() throws Exception {
    final TestingFileEnumerator fileEnumerator = new TestingFileEnumerator();
    final TestingSplitEnumeratorContext<FileSourceSplit> context = new TestingSplitEnumeratorContext<>(4);
    final ContinuousFileSplitEnumerator enumerator = createEnumerator(fileEnumerator, context);
    // register one reader, and let it request a split
    context.registerReader(2, "localhost");
    enumerator.addReader(2);
    enumerator.handleSplitRequest(2, "localhost");
    // remove the reader (like in a failure)
    context.registeredReaders().remove(2);
    // make one split available and trigger the periodic discovery
    final FileSourceSplit split = createRandomSplit();
    fileEnumerator.addSplits(split);
    context.triggerAllActions();
    assertFalse(context.getSplitAssignments().containsKey(2));
    assertThat(enumerator.snapshotState(1L).getSplits(), contains(split));
}

Example 22 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class StaticFileSplitEnumeratorTest method testSplitRequestForNonRegisteredReader.

@Test
public void testSplitRequestForNonRegisteredReader() throws Exception {
    final TestingSplitEnumeratorContext<FileSourceSplit> context = new TestingSplitEnumeratorContext<>(4);
    final FileSourceSplit split = createRandomSplit();
    final StaticFileSplitEnumerator enumerator = createEnumerator(context, split);
    enumerator.handleSplitRequest(3, "somehost");
    assertFalse(context.getSplitAssignments().containsKey(3));
    assertThat(enumerator.snapshotState(1L).getSplits(), contains(split));
}

Example 23 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class AdapterTestBase method testClosesStreamIfReaderCreationFails.

// ------------------------------------------------------------------------
@Test
public void testClosesStreamIfReaderCreationFails() throws Exception {
    // setup
    final Path testPath = new Path("testFs:///testpath-1");
    final CloseTestingInputStream in = new CloseTestingInputStream();
    final TestingFileSystem testFs = TestingFileSystem.createForFileStatus("testFs", TestingFileSystem.TestFileStatus.forFileWithStream(testPath, 1024, in));
    testFs.register();
    // test
    final BulkFormat<Integer, FileSourceSplit> adapter = wrapWithAdapter(createFormatFailingInInstantiation());
    try {
        adapter.createReader(new Configuration(), new FileSourceSplit("id", testPath, 0, 1024, 0, 1024));
    } catch (IOException ignored) {
    }
    // assertions
    assertTrue(in.closed);
    // cleanup
    testFs.unregister();
}

Example 24 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class AdapterTestBase method testReading.

private void testReading(FormatT format, int numSplits, int... recoverAfterRecords) throws IOException {
    // add the end boundary for recovery
    final int[] boundaries = Arrays.copyOf(recoverAfterRecords, recoverAfterRecords.length + 1);
    boundaries[boundaries.length - 1] = NUM_NUMBERS;
    // set a fetch size so that we get three records per fetch
    final Configuration config = new Configuration();
    config.set(StreamFormat.FETCH_IO_SIZE, new MemorySize(10));
    final BulkFormat<Integer, FileSourceSplit> adapter = wrapWithAdapter(format);
    final Queue<FileSourceSplit> splits = buildSplits(numSplits);
    final List<Integer> result = new ArrayList<>();
    FileSourceSplit currentSplit = null;
    BulkFormat.Reader<Integer> currentReader = null;
    for (int nextRecordToRecover : boundaries) {
        final FileSourceSplit toRecoverFrom = readNumbers(currentReader, currentSplit, adapter, splits, config, result, nextRecordToRecover - result.size());
        currentSplit = toRecoverFrom;
        currentReader = toRecoverFrom == null ? null : adapter.restoreReader(config, toRecoverFrom);
    }
    verifyIntListResult(result);
}

Example 25 with FileSourceSplit

use of org.apache.flink.connector.file.src.FileSourceSplit in project flink by apache.

the class AdapterTestBase method testClosesStreamIfReaderRestoreFails.

@Test
public void testClosesStreamIfReaderRestoreFails() throws Exception {
    // setup
    final Path testPath = new Path("testFs:///testpath-1");
    final CloseTestingInputStream in = new CloseTestingInputStream();
    final TestingFileSystem testFs = TestingFileSystem.createForFileStatus("testFs", TestingFileSystem.TestFileStatus.forFileWithStream(testPath, 1024, in));
    testFs.register();
    // test
    final BulkFormat<Integer, FileSourceSplit> adapter = wrapWithAdapter(createFormatFailingInInstantiation());
    final FileSourceSplit split = new FileSourceSplit("id", testPath, 0, 1024, 0, 1024, new String[0], new CheckpointedPosition(0L, 5L));
    try {
        adapter.restoreReader(new Configuration(), split);
    } catch (IOException ignored) {
    }
    // assertions
    assertTrue(in.closed);
    // cleanup
    testFs.unregister();
}