use of org.apache.flink.streaming.util.FiniteTestSource in project flink by apache.
the class OrcBulkWriterITCase method testOrcBulkWriter.
@Test
public void testOrcBulkWriter() throws Exception {
final File outDir = TEMPORARY_FOLDER.newFolder();
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
final Properties writerProps = new Properties();
writerProps.setProperty("orc.compress", "LZ4");
final OrcBulkWriterFactory<Record> factory = new OrcBulkWriterFactory<>(new RecordVectorizer(schema), writerProps, new Configuration());
env.setParallelism(1);
env.enableCheckpointing(100);
DataStream<Record> stream = env.addSource(new FiniteTestSource<>(testData), TypeInformation.of(Record.class));
stream.map(str -> str).addSink(StreamingFileSink.forBulkFormat(new Path(outDir.toURI()), factory).withBucketAssigner(new UniqueBucketAssigner<>("test")).build());
env.execute();
OrcBulkWriterTestUtil.validate(outDir, testData);
}
use of org.apache.flink.streaming.util.FiniteTestSource in project flink by apache.
the class SinkITCase method writerAndGlobalCommitterExecuteInStreamingMode.
@Ignore("FLINK-25726")
@Test
public void writerAndGlobalCommitterExecuteInStreamingMode() throws Exception {
final StreamExecutionEnvironment env = buildStreamEnv();
final FiniteTestSource<Integer> source = new FiniteTestSource<>(GLOBAL_COMMIT_QUEUE_RECEIVE_ALL_DATA, SOURCE_DATA);
env.addSource(source, IntegerTypeInfo.INT_TYPE_INFO).sinkTo(TestSink.newBuilder().setCommittableSerializer(TestSink.StringCommittableSerializer.INSTANCE).setGlobalCommitter((Supplier<Queue<String>> & Serializable) () -> GLOBAL_COMMIT_QUEUE).build());
env.execute();
// TODO: At present, for a bounded scenario, the occurrence of final checkpoint is not a
// deterministic event, so
// we do not need to verify this matter. After the final checkpoint becomes ready in the
// future,
// the verification of "end of input" would be restored.
GLOBAL_COMMIT_QUEUE.remove(END_OF_INPUT_STR);
assertThat(getSplittedGlobalCommittedData(), containsInAnyOrder(EXPECTED_GLOBAL_COMMITTED_DATA_IN_STREAMING_MODE.toArray()));
}
use of org.apache.flink.streaming.util.FiniteTestSource in project flink by apache.
the class SinkITCase method writerAndCommitterExecuteInStreamingMode.
@Test
public void writerAndCommitterExecuteInStreamingMode() throws Exception {
final StreamExecutionEnvironment env = buildStreamEnv();
final FiniteTestSource<Integer> source = new FiniteTestSource<>(COMMIT_QUEUE_RECEIVE_ALL_DATA, SOURCE_DATA);
env.addSource(source, IntegerTypeInfo.INT_TYPE_INFO).sinkTo(TestSink.newBuilder().setDefaultCommitter((Supplier<Queue<String>> & Serializable) () -> COMMIT_QUEUE).build());
env.execute();
assertThat(COMMIT_QUEUE, containsInAnyOrder(EXPECTED_COMMITTED_DATA_IN_STREAMING_MODE.toArray()));
}
use of org.apache.flink.streaming.util.FiniteTestSource in project flink by apache.
the class CompressionFactoryITCase method testWriteCompressedFile.
@Test
public void testWriteCompressedFile() throws Exception {
final File folder = TEMPORARY_FOLDER.newFolder();
final Path testPath = Path.fromLocalFile(folder);
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(100);
DataStream<String> stream = env.addSource(new FiniteTestSource<>(testData), TypeInformation.of(String.class));
stream.map(str -> str).addSink(StreamingFileSink.forBulkFormat(testPath, CompressWriters.forExtractor(new DefaultExtractor<String>()).withHadoopCompression(TEST_CODEC_NAME)).withBucketAssigner(new UniqueBucketAssigner<>("test")).build());
env.execute();
validateResults(folder, testData, new CompressionCodecFactory(configuration).getCodecByName(TEST_CODEC_NAME));
}
use of org.apache.flink.streaming.util.FiniteTestSource in project flink by apache.
the class SinkITCase method writerAndCommitterAndGlobalCommitterExecuteInStreamingMode.
@Ignore("FLINK-25726")
@Test
public void writerAndCommitterAndGlobalCommitterExecuteInStreamingMode() throws Exception {
final StreamExecutionEnvironment env = buildStreamEnv();
final FiniteTestSource<Integer> source = new FiniteTestSource<>(BOTH_QUEUE_RECEIVE_ALL_DATA, SOURCE_DATA);
env.addSource(source, IntegerTypeInfo.INT_TYPE_INFO).sinkTo(TestSink.newBuilder().setDefaultCommitter((Supplier<Queue<String>> & Serializable) () -> COMMIT_QUEUE).setGlobalCommitter((Supplier<Queue<String>> & Serializable) () -> GLOBAL_COMMIT_QUEUE).build());
env.execute();
// TODO: At present, for a bounded scenario, the occurrence of final checkpoint is not a
// deterministic event, so
// we do not need to verify this matter. After the final checkpoint becomes ready in the
// future,
// the verification of "end of input" would be restored.
GLOBAL_COMMIT_QUEUE.remove(END_OF_INPUT_STR);
assertThat(COMMIT_QUEUE, containsInAnyOrder(EXPECTED_COMMITTED_DATA_IN_STREAMING_MODE.toArray()));
assertThat(getSplittedGlobalCommittedData(), containsInAnyOrder(EXPECTED_GLOBAL_COMMITTED_DATA_IN_STREAMING_MODE.toArray()));
}
Aggregations