Examples with DisplayData org.apache.beam.sdk.transforms.display.DisplayData used on opensource projects

Example 11 with DisplayData

use of org.apache.beam.sdk.transforms.display.DisplayData in project beam by apache.

the class XmlSinkTest method testDisplayData.

@Test
public void testDisplayData() {
    XmlIO.Write<Integer> write = XmlIO.<Integer>write().to(testFilePrefix).withRootElement("bird").withRecordClass(Integer.class);
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("filenamePattern", "file-SSSSS-of-NNNNN.xml"));
    assertThat(displayData, hasDisplayItem("rootElement", "bird"));
    assertThat(displayData, hasDisplayItem("recordClass", Integer.class));
}

Example 12 with DisplayData

use of org.apache.beam.sdk.transforms.display.DisplayData in project beam by apache.

the class XmlSourceTest method testDisplayData.

@Test
public void testDisplayData() {
    DisplayData displayData = DisplayData.from(XmlIO.<Integer>read().from("foo.xml").withRootElement("bird").withRecordElement("cat").withMinBundleSize(1234).withRecordClass(Integer.class));
    assertThat(displayData, hasDisplayItem("filePattern", "foo.xml"));
    assertThat(displayData, hasDisplayItem("rootElement", "bird"));
    assertThat(displayData, hasDisplayItem("recordElement", "cat"));
    assertThat(displayData, hasDisplayItem("recordClass", Integer.class));
    assertThat(displayData, hasDisplayItem("minBundleSize", 1234));
}

Example 13 with DisplayData

use of org.apache.beam.sdk.transforms.display.DisplayData in project beam by apache.

the class WriteFilesTest method testShardedDisplayData.

@Test
public void testShardedDisplayData() {
    DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant(DefaultFilenamePolicy.fromParams(new Params().withBaseFilename(getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE)).withShardTemplate("-SS-of-NN")));
    SimpleSink<Void> sink = new SimpleSink<Void>(getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(1);
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("sink", sink.getClass()));
    assertThat(displayData, includesDisplayDataFor("sink", sink));
    assertThat(displayData, hasDisplayItem("numShards", 1));
}

Example 14 with DisplayData

use of org.apache.beam.sdk.transforms.display.DisplayData in project beam by apache.

the class TextIOWriteTest method testWriteWithWritableByteChannelFactory.

@Test
@Category(NeedsRunner.class)
public void testWriteWithWritableByteChannelFactory() throws Exception {
    Coder<String> coder = StringUtf8Coder.of();
    String outputName = "file.txt";
    ResourceId baseDir = FileSystems.matchNewResource(Files.createTempDirectory(tempFolder.getRoot().toPath(), "testwrite").toString(), true);
    PCollection<String> input = p.apply(Create.of(Arrays.asList(LINES2_ARRAY)).withCoder(coder));
    final WritableByteChannelFactory writableByteChannelFactory = new DrunkWritableByteChannelFactory();
    TextIO.Write write = TextIO.write().to(baseDir.resolve(outputName, ResolveOptions.StandardResolveOptions.RESOLVE_FILE).toString()).withoutSharding().withWritableByteChannelFactory(writableByteChannelFactory);
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "DRUNK"));
    input.apply(write);
    p.run();
    final List<String> drunkElems = new ArrayList<>(LINES2_ARRAY.length * 2 + 2);
    for (String elem : LINES2_ARRAY) {
        drunkElems.add(elem);
        drunkElems.add(elem);
    }
    assertOutputFiles(drunkElems.toArray(new String[0]), null, null, 1, baseDir.resolve(outputName + writableByteChannelFactory.getSuggestedFilenameSuffix(), ResolveOptions.StandardResolveOptions.RESOLVE_FILE), write.inner.getShardTemplate());
}

Example 15 with DisplayData

use of org.apache.beam.sdk.transforms.display.DisplayData in project beam by apache.

the class TextIOWriteTest method testWriteDisplayDataValidateThenFooter.

@Test
public void testWriteDisplayDataValidateThenFooter() {
    TextIO.Write write = TextIO.write().to("foo").withFooter("myFooter");
    DisplayData displayData = DisplayData.from(write);
    assertThat(displayData, hasDisplayItem("fileFooter", "myFooter"));
}