Examples with StreamingShardedWriteFactory org.apache.beam.runners.flink.FlinkStreamingPipelineTranslator.StreamingShardedWriteFactory used on opensource projects

Example 1 with StreamingShardedWriteFactory

use of org.apache.beam.runners.flink.FlinkStreamingPipelineTranslator.StreamingShardedWriteFactory in project beam by apache.

the class FlinkTransformOverridesTest method testRunnerDeterminedSharding.

@Test
public void testRunnerDeterminedSharding() {
    FlinkPipelineOptions options = FlinkPipelineOptions.defaults();
    options.setRunner(TestFlinkRunner.class);
    options.setFlinkMaster("[auto]");
    options.setParallelism(5);
    TestPipeline p = TestPipeline.fromOptions(options);
    StreamingShardedWriteFactory<Object, Void, Object> factory = new StreamingShardedWriteFactory<>(p.getOptions());
    WriteFiles<Object, Void, Object> original = WriteFiles.to(new TestSink(tmpFolder.toString()));
    @SuppressWarnings("unchecked") PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of()));
    AppliedPTransform<PCollection<Object>, WriteFilesResult<Void>, WriteFiles<Object, Void, Object>> originalApplication = AppliedPTransform.of("writefiles", PValues.expandInput(objs), Collections.emptyMap(), original, ResourceHints.create(), p);
    WriteFiles<Object, Void, Object> replacement = (WriteFiles<Object, Void, Object>) factory.getReplacementTransform(originalApplication).getTransform();
    assertThat(replacement, not(equalTo((Object) original)));
    assertThat(replacement.getNumShardsProvider().get(), is(10));
}