Examples with MockCommitRequest org.apache.flink.api.connector.sink2.mocks.MockCommitRequest used on opensource projects

Example 1 with MockCommitRequest

use of org.apache.flink.api.connector.sink2.mocks.MockCommitRequest in project flink by apache.

the class KafkaCommitterTest method testRetryCommittableOnRetriableError.

/**
 * Causes a network error by inactive broker and tests that a retry will happen.
 */
@Test
public void testRetryCommittableOnRetriableError() throws IOException, InterruptedException {
    Properties properties = getProperties();
    try (final KafkaCommitter committer = new KafkaCommitter(properties);
        FlinkKafkaInternalProducer<Object, Object> producer = new FlinkKafkaInternalProducer<>(properties, TRANSACTIONAL_ID);
        Recyclable<FlinkKafkaInternalProducer<Object, Object>> recyclable = new Recyclable<>(producer, p -> {
        })) {
        final MockCommitRequest<KafkaCommittable> request = new MockCommitRequest<>(new KafkaCommittable(PRODUCER_ID, EPOCH, TRANSACTIONAL_ID, recyclable));
        producer.resumeTransaction(PRODUCER_ID, EPOCH);
        committer.commit(Collections.singletonList(request));
        assertThat(request.getNumberOfRetries()).isEqualTo(1);
        assertThat(recyclable.isRecycled()).isFalse();
        // FLINK-25531: force the producer to close immediately, else it would take 1 hour
        producer.close(Duration.ZERO);
    }
}

Example 2 with MockCommitRequest

use of org.apache.flink.api.connector.sink2.mocks.MockCommitRequest in project flink by apache.

the class KafkaCommitterTest method testFailJobOnUnknownFatalError.

@Test
public void testFailJobOnUnknownFatalError() throws IOException, InterruptedException {
    Properties properties = getProperties();
    try (final KafkaCommitter committer = new KafkaCommitter(properties);
        FlinkKafkaInternalProducer<Object, Object> producer = new FlinkKafkaInternalProducer<>(properties, TRANSACTIONAL_ID);
        Recyclable<FlinkKafkaInternalProducer<Object, Object>> recyclable = new Recyclable<>(producer, p -> {
        })) {
        // will fail because transaction not started
        final MockCommitRequest<KafkaCommittable> request = new MockCommitRequest<>(new KafkaCommittable(PRODUCER_ID, EPOCH, TRANSACTIONAL_ID, recyclable));
        committer.commit(Collections.singletonList(request));
        assertThat(request.getFailedWithUnknownReason()).isInstanceOf(IllegalStateException.class);
        assertThat(request.getFailedWithUnknownReason().getMessage()).contains("Transaction was not started");
        assertThat(recyclable.isRecycled()).isTrue();
    }
}

Example 3 with MockCommitRequest

use of org.apache.flink.api.connector.sink2.mocks.MockCommitRequest in project flink by apache.

the class FileWriterBucketStateSerializerMigrationTest method testDeserializationFull.

private void testDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException, InterruptedException {
    final BucketStatePathResolver pathResolver = new BucketStatePathResolver(BASE_PATH, previousVersion);
    try {
        final java.nio.file.Path outputPath = pathResolver.getOutputPath(scenarioName);
        final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());
        // restore the state
        final FileWriterBucketState recoveredState = readBucketStateFromTemplate(scenarioName, previousVersion);
        final int noOfPendingCheckpoints = 5;
        // there are 5 checkpoint does not complete.
        final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = recoveredState.getPendingFileRecoverablesPerCheckpoint();
        Assert.assertEquals(5L, pendingFileRecoverables.size());
        final Set<String> beforeRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // before retsoring all file has "inprogress"
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = ".part-0-" + i + ".inprogress";
            assertThat(beforeRestorePaths, hasItem(startsWith(part)));
        }
        final FileWriterBucket<String> bucket = restoreBucket(recoveredState);
        Assert.assertEquals(testBucketPath, bucket.getBucketPath());
        Assert.assertEquals(noOfPendingCheckpoints, bucket.getPendingFiles().size());
        // simulates we commit the recovered pending files on the first checkpoint
        bucket.snapshotState();
        Collection<CommitRequest<FileSinkCommittable>> committables = bucket.prepareCommit(false).stream().map(MockCommitRequest::new).collect(Collectors.toList());
        FileCommitter committer = new FileCommitter(createBucketWriter());
        committer.commit(committables);
        final Set<String> afterRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)).map(file -> file.getFileName().toString()).collect(Collectors.toSet());
        // there is no "inporgress" in file name for the committed files.
        for (int i = 0; i < noOfPendingCheckpoints; i++) {
            final String part = "part-0-" + i;
            assertThat(afterRestorePaths, hasItem(part));
            afterRestorePaths.remove(part);
        }
        if (withInProgress) {
            // only the in-progress must be left
            assertThat(afterRestorePaths, iterableWithSize(1));
            // verify that the in-progress file is still there
            assertThat(afterRestorePaths, hasItem(startsWith(".part-0-" + noOfPendingCheckpoints + ".inprogress")));
        } else {
            assertThat(afterRestorePaths, empty());
        }
    } finally {
        FileUtils.deleteDirectory(pathResolver.getResourcePath(scenarioName).toFile());
    }
}

Example 4 with MockCommitRequest

use of org.apache.flink.api.connector.sink2.mocks.MockCommitRequest in project flink by apache.

the class FileCommitterTest method testCleanupInProgressFiles.

@Test
public void testCleanupInProgressFiles() throws Exception {
    StubBucketWriter stubBucketWriter = new StubBucketWriter();
    FileCommitter fileCommitter = new FileCommitter(stubBucketWriter);
    MockCommitRequest<FileSinkCommittable> fileSinkCommittable = new MockCommitRequest<>(new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable()));
    fileCommitter.commit(Collections.singletonList(fileSinkCommittable));
    assertEquals(0, stubBucketWriter.getRecoveredPendingFiles().size());
    assertEquals(1, stubBucketWriter.getNumCleanUp());
    assertEquals(0, fileSinkCommittable.getNumberOfRetries());
}

Example 5 with MockCommitRequest

use of org.apache.flink.api.connector.sink2.mocks.MockCommitRequest in project flink by apache.

the class FileCommitterTest method testCommitMultiple.

@Test
public void testCommitMultiple() throws Exception {
    StubBucketWriter stubBucketWriter = new StubBucketWriter();
    FileCommitter fileCommitter = new FileCommitter(stubBucketWriter);
    Collection<CommitRequest<FileSinkCommittable>> committables = Stream.of(new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestPendingFileRecoverable()), new FileSinkCommittable("0", new FileSinkTestUtils.TestInProgressFileRecoverable())).map(MockCommitRequest::new).collect(Collectors.toList());
    fileCommitter.commit(committables);
    assertEquals(3, stubBucketWriter.getRecoveredPendingFiles().size());
    assertEquals(2, stubBucketWriter.getNumCleanUp());
    stubBucketWriter.getRecoveredPendingFiles().forEach(pendingFile -> assertTrue(pendingFile.isCommitted()));
    assertTrue(committables.stream().allMatch(c -> c.getNumberOfRetries() == 0));
}