Examples with FieldAccessDescriptor org.apache.beam.sdk.schemas.FieldAccessDescriptor used on opensource projects

Example 6 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class BeamCalcRelTest method testNoFieldAccess.

@Test
public void testNoFieldAccess() throws IllegalAccessException {
    String sql = "SELECT 1 FROM ORDER_DETAILS_BOUNDED";
    PCollection<Row> rows = compilePipeline(sql, pipeline);
    final NodeGetter nodeGetter = new NodeGetter(rows);
    pipeline.traverseTopologically(nodeGetter);
    ParDo.MultiOutput<Row, Row> pardo = (ParDo.MultiOutput<Row, Row>) nodeGetter.producer.getTransform();
    PCollection<Row> input = (PCollection<Row>) Iterables.getOnlyElement(nodeGetter.producer.getInputs().values());
    DoFnSchemaInformation info = ParDo.getDoFnSchemaInformation(pardo.getFn(), input);
    FieldAccessDescriptor fieldAccess = info.getFieldAccessDescriptor();
    Assert.assertFalse(fieldAccess.getAllFields());
    Assert.assertTrue(fieldAccess.getFieldsAccessed().isEmpty());
    Assert.assertTrue(fieldAccess.getNestedFieldsAccessed().isEmpty());
    pipeline.run().waitUntilFinish();
}

Example 7 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class BeamZetaSqlCalcRelTest method testSingleFieldAccess.

@Test
public void testSingleFieldAccess() throws IllegalAccessException {
    String sql = "SELECT Key FROM KeyValue";
    PCollection<Row> rows = compile(sql);
    final NodeGetter nodeGetter = new NodeGetter(rows);
    pipeline.traverseTopologically(nodeGetter);
    ParDo.MultiOutput<Row, Row> pardo = (ParDo.MultiOutput<Row, Row>) nodeGetter.producer.getTransform();
    PCollection<Row> input = (PCollection<Row>) Iterables.getOnlyElement(nodeGetter.producer.getInputs().values());
    DoFnSchemaInformation info = ParDo.getDoFnSchemaInformation(pardo.getFn(), input);
    FieldAccessDescriptor fieldAccess = info.getFieldAccessDescriptor();
    Assert.assertTrue(fieldAccess.referencesSingleField());
    Assert.assertEquals("Key", Iterables.getOnlyElement(fieldAccess.fieldNamesAccessed()));
    pipeline.run().waitUntilFinish();
}

Example 8 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testSelectMapOfArray.

@Test
public void testSelectMapOfArray() {
    FieldAccessDescriptor fieldAccessDescriptor = FieldAccessDescriptor.withFieldNames("map.field1").resolve(MAP_ARRAY_SCHEMA);
    Schema outputSchema = SelectHelpers.getOutputSchema(MAP_ARRAY_SCHEMA, fieldAccessDescriptor);
    Schema expectedSchema = Schema.builder().addMapField("field1", FieldType.INT32, FieldType.array(FieldType.STRING)).build();
    assertEquals(expectedSchema, outputSchema);
    Row row = selectRow(MAP_ARRAY_SCHEMA, fieldAccessDescriptor, MAP_ARRAY_ROW);
    Row expectedRow = Row.withSchema(expectedSchema).addValue(ImmutableMap.of(1, ImmutableList.of("first"))).build();
    assertEquals(expectedRow, row);
}

Example 9 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testSelectMapOfRowSelectSingle.

@Test
public void testSelectMapOfRowSelectSingle() {
    FieldAccessDescriptor fieldAccessDescriptor = FieldAccessDescriptor.withFieldNames("map{}.field1").resolve(MAP_SCHEMA);
    Schema outputSchema = SelectHelpers.getOutputSchema(MAP_SCHEMA, fieldAccessDescriptor);
    Schema.builder().addStringField("field1").build();
    Schema expectedSchema = Schema.builder().addMapField("field1", FieldType.INT32, FieldType.STRING).build();
    assertEquals(expectedSchema, outputSchema);
    Row row = selectRow(MAP_SCHEMA, fieldAccessDescriptor, MAP_ROW);
    Row expectedRow = Row.withSchema(expectedSchema).addValue(ImmutableMap.of(1, "first")).build();
    assertEquals(expectedRow, row);
}

Example 10 with FieldAccessDescriptor

use of org.apache.beam.sdk.schemas.FieldAccessDescriptor in project beam by apache.

the class SelectHelpersTest method testSelectFieldOfRecordOrRecord.

@Test
public void testSelectFieldOfRecordOrRecord() {
    Schema f1 = Schema.builder().addInt64Field("f0").build();
    Schema f2 = Schema.builder().addRowField("f1", f1).build();
    Schema f3 = Schema.builder().addRowField("f2", f2).build();
    Schema f4 = Schema.builder().addRowField("f3", f3).build();
    // {"f0": 42}
    Row r1 = Row.withSchema(f1).addValue(42L).build();
    // {"f1": {"f0": 42}}
    Row r2 = Row.withSchema(f2).addValue(r1).build();
    // {"f2": {"f1": {"f0": 42}}}
    Row r3 = Row.withSchema(f3).addValue(r2).build();
    // {"f3": {"f2": {"f1": {"f0": 42}}}}
    Row r4 = Row.withSchema(f4).addValue(r3).build();
    FieldAccessDescriptor fieldAccessDescriptor = FieldAccessDescriptor.withFieldNames("f3.f2").resolve(f4);
    Schema outputSchema = SelectHelpers.getOutputSchema(f4, fieldAccessDescriptor);
    Row out = selectRow(f4, fieldAccessDescriptor, r4);
    assertEquals(f3, outputSchema);
    assertEquals(r3, out);
}