Examples with FieldAccessDescriptor org.apache.beam.sdk.schemas.FieldAccessDescriptor used on opensource projects

Example 61 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testsSimpleSelectMultiple.

@Test
public void testsSimpleSelectMultiple() {
    FieldAccessDescriptor fieldAccessDescriptor = FieldAccessDescriptor.withFieldNames("field1", "field3").resolve(FLAT_SCHEMA);
    Schema outputSchema = SelectHelpers.getOutputSchema(FLAT_SCHEMA, fieldAccessDescriptor);
    Schema expectedSchema = Schema.builder().addStringField("field1").addDoubleField("field3").build();
    assertEquals(expectedSchema, outputSchema);
    Row row = selectRow(FLAT_SCHEMA, fieldAccessDescriptor, FLAT_ROW);
    Row expectedRow = Row.withSchema(expectedSchema).addValues("first", 3.14).build();
    assertEquals(expectedRow, row);
}

Example 62 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testSelectIterableOfRowIterable.

@Test
public void testSelectIterableOfRowIterable() {
    FieldAccessDescriptor fieldAccessDescriptor = FieldAccessDescriptor.withFieldNames("iterOfRowIter[][].field1").resolve(ITERABLE_SCHEMA);
    Schema outputSchema = SelectHelpers.getOutputSchema(ITERABLE_SCHEMA, fieldAccessDescriptor);
    Schema expectedSchema = Schema.builder().addIterableField("field1", FieldType.iterable(FieldType.STRING)).build();
    assertEquals(expectedSchema, outputSchema);
    Row row = selectRow(ITERABLE_SCHEMA, fieldAccessDescriptor, ITERABLE_ROW);
    Row expectedRow = Row.withSchema(expectedSchema).addIterable(ImmutableList.of(ImmutableList.of("first"), ImmutableList.of("first"))).build();
    assertEquals(expectedRow, row);
}

Example 63 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testSelectNullableNestedRowMap.

@Test
public void testSelectNullableNestedRowMap() {
    FieldAccessDescriptor fieldAccessDescriptor1 = FieldAccessDescriptor.withFieldNames("nestedMap.field1").resolve(NESTED_NULLABLE_SCHEMA);
    Row out1 = selectRow(NESTED_NULLABLE_SCHEMA, fieldAccessDescriptor1, Row.nullRow(NESTED_NULLABLE_SCHEMA));
    assertNull(out1.getValue(0));
    FieldAccessDescriptor fieldAccessDescriptor2 = FieldAccessDescriptor.withFieldNames("nestedMap.*").resolve(NESTED_NULLABLE_SCHEMA);
    Row out2 = selectRow(NESTED_NULLABLE_SCHEMA, fieldAccessDescriptor2, Row.nullRow(NESTED_NULLABLE_SCHEMA));
    assertEquals(Collections.nCopies(4, null), out2.getValues());
}

Example 64 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class ParDoSchemaTest method testUnmatchedSchema.

@Test
@Category(NeedsRunner.class)
public void testUnmatchedSchema() {
    List<MyPojo> pojoList = Lists.newArrayList(new MyPojo("a", 1), new MyPojo("b", 2), new MyPojo("c", 3));
    Schema schema = Schema.builder().addStringField("string_field").addInt32Field("integer_field").build();
    thrown.expect(IllegalArgumentException.class);
    pipeline.apply(Create.of(pojoList).withSchema(schema, TypeDescriptor.of(MyPojo.class), o -> Row.withSchema(schema).addValues(o.stringField, o.integerField).build(), r -> new MyPojo(r.getString("string_field"), r.getInt32("integer_field")))).apply(ParDo.of(new DoFn<MyPojo, Void>() {

        @FieldAccess("a")
        FieldAccessDescriptor fieldAccess = FieldAccessDescriptor.withFieldNames("baad");

        @ProcessElement
        public void process(@FieldAccess("a") Row row) {
        }
    }));
    pipeline.run();
}

Example 65 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class DoFnSignaturesTest method testFieldAccess.

@Test
public void testFieldAccess() throws IllegalAccessException {
    FieldAccessDescriptor descriptor = FieldAccessDescriptor.withFieldNames("foo", "bar");
    DoFn<String, String> doFn = new DoFn<String, String>() {

        @FieldAccess("foo")
        final FieldAccessDescriptor fieldAccess = descriptor;

        @ProcessElement
        public void process(@FieldAccess("foo") @Element Row row) {
        }
    };
    DoFnSignature sig = DoFnSignatures.getSignature(doFn.getClass());
    assertThat(sig.fieldAccessDeclarations().get("foo"), notNullValue());
    Field field = sig.fieldAccessDeclarations().get("foo").field();
    assertThat(field.getName(), equalTo("fieldAccess"));
    assertThat(field.get(doFn), equalTo(descriptor));
    assertFalse(sig.processElement().getSchemaElementParameters().isEmpty());
}