Examples with Row org.apache.beam.sdk.values.Row used on opensource projects

Example 36 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class FieldAccessVisitorTest method testFieldAccessTwoKnownMainInputs.

@Test
public void testFieldAccessTwoKnownMainInputs() {
    Pipeline p = Pipeline.create();
    FieldAccessVisitor fieldAccessVisitor = new FieldAccessVisitor();
    Schema schema = Schema.of(Field.of("field1", FieldType.STRING), Field.of("field2", FieldType.STRING), Field.of("field3", FieldType.STRING));
    PCollection<Row> source = p.apply(Create.of(Row.withSchema(schema).addValues("foo", "bar", "baz").build())).setRowSchema(schema);
    source.apply(new FieldAccessTransform(FieldAccessDescriptor.withFieldNames("field1")));
    source.apply(new FieldAccessTransform(FieldAccessDescriptor.withFieldNames("field2")));
    p.traverseTopologically(fieldAccessVisitor);
    FieldAccessDescriptor fieldAccess = fieldAccessVisitor.getPCollectionFieldAccess().get(source);
    assertFalse(fieldAccess.getAllFields());
    assertThat(fieldAccess.fieldNamesAccessed(), containsInAnyOrder("field1", "field2"));
}

Example 37 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class FieldAccessVisitorTest method testFieldAccessKnownMainInput.

@Test
public void testFieldAccessKnownMainInput() {
    Pipeline p = Pipeline.create();
    FieldAccessVisitor fieldAccessVisitor = new FieldAccessVisitor();
    Schema schema = Schema.of(Field.of("field1", FieldType.STRING), Field.of("field2", FieldType.STRING));
    PCollection<Row> source = p.apply(Create.of(Row.withSchema(schema).addValues("foo", "bar").build())).setRowSchema(schema);
    source.apply(new FieldAccessTransform(FieldAccessDescriptor.withFieldNames("field1")));
    p.traverseTopologically(fieldAccessVisitor);
    FieldAccessDescriptor fieldAccess = fieldAccessVisitor.getPCollectionFieldAccess().get(source);
    assertFalse(fieldAccess.getAllFields());
    assertThat(fieldAccess.fieldNamesAccessed(), containsInAnyOrder("field1"));
}

Example 38 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class ToJson method expand.

@Override
public PCollection<String> expand(PCollection<T> rows) {
    Schema inputSchema = rows.getSchema();
    // Throw exception if this schema is not supported by RowJson
    RowJson.verifySchemaSupported(inputSchema);
    SerializableFunction<T, Row> toRow = rows.getToRowFunction();
    return rows.apply(ParDo.of(new DoFn<T, String>() {

        @ProcessElement
        public void processElement(ProcessContext context) {
            context.output(rowToJson(objectMapper(inputSchema), toRow.apply(context.element())));
        }
    }));
}

Example 39 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testEncodingPositionReorderFields.

@Test
public void testEncodingPositionReorderFields() throws Exception {
    Schema schema1 = Schema.builder().addNullableField("f_int32", FieldType.INT32).addNullableField("f_string", FieldType.STRING).build();
    Schema schema2 = Schema.builder().addNullableField("f_string", FieldType.STRING).addNullableField("f_int32", FieldType.INT32).build();
    schema2.setEncodingPositions(ImmutableMap.of("f_int32", 0, "f_string", 1));
    Row row = Row.withSchema(schema1).withFieldValue("f_int32", 42).withFieldValue("f_string", "hello world!").build();
    Row expected = Row.withSchema(schema2).withFieldValue("f_int32", 42).withFieldValue("f_string", "hello world!").build();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    RowCoder.of(schema1).encode(row, os);
    Row decoded = RowCoder.of(schema2).decode(new ByteArrayInputStream(os.toByteArray()));
    assertEquals(expected, decoded);
}

Example 40 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testLogicalTypeInCollection.

@Test
public void testLogicalTypeInCollection() throws Exception {
    EnumerationType enumeration = EnumerationType.create("one", "two", "three");
    Schema schema = Schema.builder().addArrayField("f_enum_array", FieldType.logicalType(enumeration)).build();
    Row row = Row.withSchema(schema).addArray(enumeration.valueOf("two"), enumeration.valueOf("three")).build();
    CoderProperties.coderDecodeEncodeEqual(RowCoder.of(schema), row);
}