Examples with TupleToIntegerBucketer org.apache.flink.streaming.api.functions.sink.filesystem.TestUtils.TupleToIntegerBucketer used on opensource projects

Example 1 with TupleToIntegerBucketer

use of org.apache.flink.streaming.api.functions.sink.filesystem.TestUtils.TupleToIntegerBucketer in project flink by apache.

the class LocalStreamingFileSinkTest method testClosingWithCustomizedBucketer.

@Test
public void testClosingWithCustomizedBucketer() throws Exception {
    final File outDir = TEMP_FOLDER.newFolder();
    final long partMaxSize = 2L;
    final long inactivityInterval = 100L;
    final RollingPolicy<Tuple2<String, Integer>, Integer> rollingPolicy = DefaultRollingPolicy.builder().withMaxPartSize(new MemorySize(partMaxSize)).withRolloverInterval(Duration.ofMillis(inactivityInterval)).withInactivityInterval(Duration.ofMillis(inactivityInterval)).build();
    try (OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness = TestUtils.createCustomizedRescalingTestSink(outDir, 1, 0, 100L, new TupleToIntegerBucketer(), new Tuple2Encoder(), rollingPolicy, new DefaultBucketFactoryImpl<>())) {
        testHarness.setup();
        testHarness.open();
        testHarness.setProcessingTime(0L);
        testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
        testHarness.processElement(new StreamRecord<>(Tuple2.of("test2", 2), 1L));
        TestUtils.checkLocalFs(outDir, 2, 0);
        // this is to check the inactivity threshold
        testHarness.setProcessingTime(101L);
        TestUtils.checkLocalFs(outDir, 2, 0);
        testHarness.processElement(new StreamRecord<>(Tuple2.of("test3", 3), 1L));
        TestUtils.checkLocalFs(outDir, 3, 0);
        testHarness.snapshot(0L, 1L);
        TestUtils.checkLocalFs(outDir, 3, 0);
        testHarness.notifyOfCompletedCheckpoint(0L);
        TestUtils.checkLocalFs(outDir, 0, 3);
        testHarness.processElement(new StreamRecord<>(Tuple2.of("test4", 4), 10L));
        TestUtils.checkLocalFs(outDir, 1, 3);
        testHarness.snapshot(1L, 0L);
        testHarness.notifyOfCompletedCheckpoint(1L);
    }
    // at close all files moved to final.
    TestUtils.checkLocalFs(outDir, 0, 4);
    // check file content and bucket ID.
    Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
    for (Map.Entry<File, String> fileContents : contents.entrySet()) {
        Integer bucketId = Integer.parseInt(fileContents.getKey().getParentFile().getName());
        Assert.assertTrue(bucketId >= 1 && bucketId <= 4);
        Assert.assertEquals(String.format("test%d@%d\n", bucketId, bucketId), fileContents.getValue());
    }
}