Examples with Params org.apache.beam.sdk.io.DefaultFilenamePolicy.Params used on opensource projects

Example 1 with Params

use of org.apache.beam.sdk.io.DefaultFilenamePolicy.Params in project beam by apache.

the class WriteFilesTest method testShardedDisplayData.

@Test
public void testShardedDisplayData() {
    DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant(DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE)).withShardTemplate("-SS-of-NN")));
    SimpleSink<Void> sink = new SimpleSink<Void>(getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(1);
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
    assertThat(displayData, includesDisplayDataFor("sink", sink));
    assertThat(displayData, hasDisplayItem("numShards", 1));
}

Example 2 with Params

use of org.apache.beam.sdk.io.DefaultFilenamePolicy.Params in project beam by apache.

the class WriteFilesTest method testCustomShardStrategyDisplayData.

@Test
public void testCustomShardStrategyDisplayData() {
    DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant(DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE)).withShardTemplate("-SS-of-NN")));
    SimpleSink<Void> sink = new SimpleSink<Void>(getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    WriteFiles<String, ?, String> write = WriteFiles.to(sink).withSharding(new PTransform<PCollection<String>, PCollectionView<Integer>>() {

        @Override
        public PCollectionView<Integer> expand(PCollection<String> input) {
            return null;
        }

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("spam", "ham"));
        }
    });
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
    assertThat(displayData, includesDisplayDataFor("sink", sink));
    assertThat(displayData, hasDisplayItem("spam", "ham"));
}

Example 3 with Params

use of org.apache.beam.sdk.io.DefaultFilenamePolicy.Params in project beam by apache.

the class WriteFilesTest method testDisplayData.

@Test
public void testDisplayData() {
    DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant(DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE)).withShardTemplate("-SS-of-NN")));
    SimpleSink<Void> sink = new SimpleSink<Void>(getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    WriteFiles<String, ?, String> write = WriteFiles.to(sink);
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
    assertThat(displayData, includesDisplayDataFor("sink", sink));
}