Examples with CommittableMessage org.apache.flink.streaming.api.connector.sink2.CommittableMessage used on opensource projects

Example 11 with CommittableMessage

use of org.apache.flink.streaming.api.connector.sink2.CommittableMessage in project flink by apache.

the class CompactorOperatorTest method testPassthrough.

@Test
public void testPassthrough() throws Exception {
    FileCompactor fileCompactor = new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
    CompactorOperator compactor = createTestOperator(fileCompactor);
    try (OneInputStreamOperatorTestHarness<CompactorRequest, CommittableMessage<FileSinkCommittable>> harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
        harness.setup();
        harness.open();
        FileSinkCommittable cleanupInprogressRequest = cleanupInprogress("0", "0", 1);
        FileSinkCommittable cleanupPathRequest = cleanupPath("0", "1");
        harness.processElement(request("0", null, Collections.singletonList(cleanupInprogressRequest)));
        harness.processElement(request("0", null, Collections.singletonList(cleanupPathRequest)));
        Assert.assertEquals(0, harness.extractOutputValues().size());
        harness.prepareSnapshotPreBarrier(1);
        harness.snapshot(1, 1L);
        harness.notifyOfCompletedCheckpoint(1);
        compactor.getAllTasksFuture().join();
        Assert.assertEquals(0, harness.extractOutputValues().size());
        harness.prepareSnapshotPreBarrier(2);
        List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
        Assert.assertEquals(3, results.size());
        SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0)).hasPendingCommittables(2);
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(1)).hasCommittable(cleanupInprogressRequest);
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(2)).hasCommittable(cleanupPathRequest);
    }
}

Example 12 with CommittableMessage

use of org.apache.flink.streaming.api.connector.sink2.CommittableMessage in project flink by apache.

the class CompactorOperatorTest method testRestore.

@Test
public void testRestore() throws Exception {
    FileCompactor fileCompactor = new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
    CompactorOperator compactor = createTestOperator(fileCompactor);
    OperatorSubtaskState state;
    try (OneInputStreamOperatorTestHarness<CompactorRequest, CommittableMessage<FileSinkCommittable>> harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
        harness.setup();
        harness.open();
        harness.processElement(request("0", Arrays.asList(committable("0", ".0", 5), committable("0", ".1", 5)), null));
        harness.snapshot(1, 1L);
        harness.processElement(request("0", Arrays.asList(committable("0", ".2", 5), committable("0", ".3", 5)), null));
        harness.notifyOfCompletedCheckpoint(1);
        // request 1 is submitted and request 2 is pending
        state = harness.snapshot(2, 2L);
    }
    compactor = createTestOperator(fileCompactor);
    try (OneInputStreamOperatorTestHarness<CompactorRequest, CommittableMessage<FileSinkCommittable>> harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
        harness.setup();
        harness.initializeState(state);
        harness.open();
        // request 1 should be submitted
        compactor.getAllTasksFuture().join();
        harness.prepareSnapshotPreBarrier(3);
        // the result of request 1 should be emitted
        Assert.assertEquals(4, harness.extractOutputValues().size());
        harness.snapshot(3, 3L);
        harness.notifyOfCompletedCheckpoint(3L);
        // request 2 should be submitted
        compactor.getAllTasksFuture().join();
        harness.prepareSnapshotPreBarrier(4);
        // the result of request 2 should be emitted
        Assert.assertEquals(8, harness.extractOutputValues().size());
        // 1summary+1compacted+2cleanup * 2
        List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
        Assert.assertEquals(8, results.size());
        SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0)).hasPendingCommittables(3);
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(1)).hasCommittable(committable("0", "compacted-0", 10));
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(2)).hasCommittable(cleanupPath("0", ".0"));
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(3)).hasCommittable(cleanupPath("0", ".1"));
        SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(4)).hasPendingCommittables(3);
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(5)).hasCommittable(committable("0", "compacted-2", 10));
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(6)).hasCommittable(cleanupPath("0", ".2"));
        SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(7)).hasCommittable(cleanupPath("0", ".3"));
    }
}

Example 13 with CommittableMessage

use of org.apache.flink.streaming.api.connector.sink2.CommittableMessage in project flink by apache.

the class CompactCoordinatorTest method testPassthrough.

@Test
public void testPassthrough() throws Exception {
    FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
    CompactCoordinator coordinator = new CompactCoordinator(strategy, getTestCommittableSerializer());
    try (OneInputStreamOperatorTestHarness<CommittableMessage<FileSinkCommittable>, CompactorRequest> harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
        harness.setup();
        harness.open();
        FileSinkCommittable cleanupToPassthrough = cleanupInprogress("0", ".0", 1);
        FileSinkCommittable sizeUnavailableToPassthrough = committable("0", ".1", -1);
        FileSinkCommittable pathNotHidToPassThrough = committable("0", "2", -1);
        FileSinkCommittable normalCommittable = committable("0", ".3", 10);
        harness.processElement(message(cleanupToPassthrough));
        harness.processElement(message(sizeUnavailableToPassthrough));
        harness.processElement(message(pathNotHidToPassThrough));
        harness.processElement(message(normalCommittable));
        List<CompactorRequest> results = harness.extractOutputValues();
        Assert.assertEquals(1, results.size());
        assertToCompact(results.get(0), normalCommittable);
        assertToPassthrough(results.get(0), cleanupToPassthrough, sizeUnavailableToPassthrough, pathNotHidToPassThrough);
    }
}

Example 14 with CommittableMessage

use of org.apache.flink.streaming.api.connector.sink2.CommittableMessage in project flink by apache.

the class CompactCoordinatorTest method testRestore.

@Test
public void testRestore() throws Exception {
    FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
    CompactCoordinator coordinator = new CompactCoordinator(strategy, getTestCommittableSerializer());
    FileSinkCommittable committable0 = committable("0", ".0", 5);
    FileSinkCommittable committable1 = committable("0", ".1", 6);
    FileSinkCommittable committable2 = committable("0", ".2", 5);
    FileSinkCommittable committable3 = committable("1", ".0", 5);
    OperatorSubtaskState state;
    try (OneInputStreamOperatorTestHarness<CommittableMessage<FileSinkCommittable>, CompactorRequest> harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
        harness.setup();
        harness.open();
        harness.processElement(message(committable0));
        Assert.assertEquals(0, harness.extractOutputValues().size());
        harness.prepareSnapshotPreBarrier(1);
        state = harness.snapshot(1, 1);
    }
    coordinator = new CompactCoordinator(strategy, getTestCommittableSerializer());
    try (OneInputStreamOperatorTestHarness<CommittableMessage<FileSinkCommittable>, CompactorRequest> harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
        harness.setup();
        harness.initializeState(state);
        harness.open();
        harness.processElement(message(committable1));
        Assert.assertEquals(1, harness.extractOutputValues().size());
        harness.processElement(message(committable2));
        harness.processElement(message(committable3));
        Assert.assertEquals(1, harness.extractOutputValues().size());
        harness.endInput();
        List<CompactorRequest> results = harness.extractOutputValues();
        Assert.assertEquals(3, results.size());
        assertToCompact(results.get(0), committable0, committable1);
        assertToCompact(results.get(1), committable2);
        assertToCompact(results.get(2), committable3);
    }
}

Example 15 with CommittableMessage

use of org.apache.flink.streaming.api.connector.sink2.CommittableMessage in project flink by apache.

the class CompactCoordinatorTest method testRestoreWithChangedStrategy.

@Test
public void testRestoreWithChangedStrategy() throws Exception {
    FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(100).build();
    CompactCoordinator coordinator = new CompactCoordinator(strategy, getTestCommittableSerializer());
    FileSinkCommittable committable0 = committable("0", ".0", 5);
    FileSinkCommittable committable1 = committable("0", ".1", 6);
    FileSinkCommittable committable2 = committable("0", ".2", 7);
    FileSinkCommittable committable3 = committable("0", ".3", 8);
    FileSinkCommittable committable4 = committable("0", ".4", 9);
    FileSinkCommittable committable5 = committable("0", ".5", 2);
    OperatorSubtaskState state;
    try (OneInputStreamOperatorTestHarness<CommittableMessage<FileSinkCommittable>, CompactorRequest> harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
        harness.setup();
        harness.open();
        harness.processElement(message(committable0));
        harness.processElement(message(committable1));
        harness.processElement(message(committable2));
        harness.processElement(message(committable3));
        harness.processElement(message(committable4));
        harness.prepareSnapshotPreBarrier(1);
        state = harness.snapshot(1, 1);
        Assert.assertEquals(0, harness.extractOutputValues().size());
    }
    FileCompactStrategy changedStrategy = Builder.newBuilder().setSizeThreshold(10).build();
    CompactCoordinator changedCoordinator = new CompactCoordinator(changedStrategy, getTestCommittableSerializer());
    try (OneInputStreamOperatorTestHarness<CommittableMessage<FileSinkCommittable>, CompactorRequest> harness = new OneInputStreamOperatorTestHarness<>(changedCoordinator)) {
        harness.setup();
        harness.initializeState(state);
        harness.open();
        Assert.assertEquals(2, harness.extractOutputValues().size());
        harness.processElement(message(committable5));
        List<CompactorRequest> results = harness.extractOutputValues();
        Assert.assertEquals(3, results.size());
        assertToCompact(results.get(0), committable0, committable1);
        assertToCompact(results.get(1), committable2, committable3);
        assertToCompact(results.get(2), committable4, committable5);
    }
}