Examples with FileSinkCommittable org.apache.flink.connector.file.sink.FileSinkCommittable used on opensource projects

Example 16 with FileSinkCommittable

use of org.apache.flink.connector.file.sink.FileSinkCommittable in project flink by apache.

the class FileWriterBucketTest method testOnCheckpointNoPendingRecoverable.

@Test
public void testOnCheckpointNoPendingRecoverable() throws IOException {
    File outDir = TEMP_FOLDER.newFolder();
    Path path = new Path(outDir.toURI());
    TestRecoverableWriter recoverableWriter = getRecoverableWriter(path);
    FileWriterBucket<String> bucket = createBucket(recoverableWriter, path, DEFAULT_ROLLING_POLICY, OutputFileConfig.builder().build());
    bucket.write("test-element", 0);
    List<FileSinkCommittable> fileSinkCommittables = bucket.prepareCommit(false);
    FileWriterBucketState bucketState = bucket.snapshotState();
    compareNumberOfPendingAndInProgress(fileSinkCommittables, 0, 0);
    assertEquals(BUCKET_ID, bucketState.getBucketId());
    assertEquals(path, bucketState.getBucketPath());
    assertNotNull("The bucket should have in-progress recoverable", bucketState.getInProgressFileRecoverable());
}

Example 17 with FileSinkCommittable

use of org.apache.flink.connector.file.sink.FileSinkCommittable in project flink by apache.

the class FileWriterBucketTest method testOnCheckpointMultiplePendingFiles.

@Test
public void testOnCheckpointMultiplePendingFiles() throws IOException {
    File outDir = TEMP_FOLDER.newFolder();
    Path path = new Path(outDir.toURI());
    TestRecoverableWriter recoverableWriter = getRecoverableWriter(path);
    FileWriterBucket<String> bucket = createBucket(recoverableWriter, path, EACH_ELEMENT_ROLLING_POLICY, OutputFileConfig.builder().build());
    bucket.write("test-element", 0);
    bucket.write("test-element", 0);
    bucket.write("test-element", 0);
    List<FileSinkCommittable> fileSinkCommittables = bucket.prepareCommit(false);
    FileWriterBucketState bucketState = bucket.snapshotState();
    // The last element would not roll
    compareNumberOfPendingAndInProgress(fileSinkCommittables, 2, 0);
    assertEquals(BUCKET_ID, bucketState.getBucketId());
    assertEquals(path, bucketState.getBucketPath());
    assertNotNull("The bucket should not have in-progress recoverable", bucketState.getInProgressFileRecoverable());
}

Example 18 with FileSinkCommittable

use of org.apache.flink.connector.file.sink.FileSinkCommittable in project flink by apache.

the class FileWriterBucketTest method testTableRollingOnProcessingTime.

@Test
public void testTableRollingOnProcessingTime() throws IOException {
    File outDir = TEMP_FOLDER.newFolder();
    Path path = new Path(outDir.toURI());
    FileSystemTableSink.TableRollingPolicy tableRollingPolicy = new FileSystemTableSink.TableRollingPolicy(false, Long.MAX_VALUE, Duration.ofMillis(20).toMillis(), Duration.ofMillis(10).toMillis());
    TestRecoverableWriter recoverableWriter = getRecoverableWriter(path);
    FileWriterBucket<RowData> bucket = createRowDataBucket(recoverableWriter, path, tableRollingPolicy, OutputFileConfig.builder().build());
    bucket.write(GenericRowData.of(StringData.fromString("test-element")), 11);
    bucket.write(GenericRowData.of(StringData.fromString("test-element")), 12);
    bucket.onProcessingTime(21);
    assertNotNull("The bucket should not roll since interval and inactivity not reached", bucket.getInProgressPart());
    bucket.onProcessingTime(22);
    assertNull("The bucket should roll since inactivity is reached", bucket.getInProgressPart());
    bucket.write(GenericRowData.of(StringData.fromString("test-element")), 11);
    bucket.write(GenericRowData.of(StringData.fromString("test-element")), 21);
    bucket.onProcessingTime(30);
    assertNotNull("The bucket should not roll since interval and inactivity not reached", bucket.getInProgressPart());
    bucket.onProcessingTime(31);
    assertNull("The bucket should roll since interval is reached", bucket.getInProgressPart());
    List<FileSinkCommittable> fileSinkCommittables = bucket.prepareCommit(false);
    compareNumberOfPendingAndInProgress(fileSinkCommittables, 2, 0);
}

Example 19 with FileSinkCommittable

use of org.apache.flink.connector.file.sink.FileSinkCommittable in project flink by apache.

the class FileWriterBucketTest method compareNumberOfPendingAndInProgress.

private void compareNumberOfPendingAndInProgress(List<FileSinkCommittable> fileSinkCommittables, int expectedPendingFiles, int expectedInProgressFiles) {
    int numPendingFiles = 0;
    int numInProgressFiles = 0;
    for (FileSinkCommittable committable : fileSinkCommittables) {
        if (committable.getPendingFile() != null) {
            numPendingFiles++;
        }
        if (committable.getInProgressFileToCleanup() != null) {
            numInProgressFiles++;
        }
    }
    assertEquals(expectedPendingFiles, numPendingFiles);
    assertEquals(expectedInProgressFiles, numInProgressFiles);
}

Example 20 with FileSinkCommittable

use of org.apache.flink.connector.file.sink.FileSinkCommittable in project flink by apache.

the class FileCommitterTest method testCleanupInProgressFiles.

@Test
public void testCleanupInProgressFiles() throws Exception {
    StubBucketWriter stubBucketWriter = new StubBucketWriter();
    FileCommitter fileCommitter = new FileCommitter(stubBucketWriter);
    MockCommitRequest<FileSinkCommittable> fileSinkCommittable = new MockCommitRequest<>(new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable()));
    fileCommitter.commit(Collections.singletonList(fileSinkCommittable));
    assertEquals(0, stubBucketWriter.getRecoveredPendingFiles().size());
    assertEquals(1, stubBucketWriter.getNumCleanUp());
    assertEquals(0, fileSinkCommittable.getNumberOfRetries());
}