Examples with Row org.apache.beam.sdk.values.Row used on opensource projects

Example 46 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testNestedLogicalTypes.

@Test
public void testNestedLogicalTypes() throws Exception {
    EnumerationType enumeration = EnumerationType.create("one", "two", "three");
    Schema schema = Schema.builder().addLogicalTypeField("f_nested_logical_type", new NestedLogicalType(enumeration)).build();
    Row row = Row.withSchema(schema).addValue("two").build();
    CoderProperties.coderDecodeEncodeEqual(RowCoder.of(schema), row);
}

Example 47 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testConsistentWithEqualsArrayOfArrayOfBytes.

@Test
public void testConsistentWithEqualsArrayOfArrayOfBytes() throws Exception {
    FieldType fieldType = FieldType.array(FieldType.array(FieldType.BYTES));
    Schema schema = Schema.of(Schema.Field.of("f1", fieldType));
    RowCoder coder = RowCoder.of(schema);
    List<byte[]> innerList1 = Collections.singletonList(new byte[] { 1, 2, 3, 4 });
    List<List<byte[]>> list1 = Collections.singletonList(innerList1);
    Row row1 = Row.withSchema(schema).addValue(list1).build();
    List<byte[]> innerList2 = Collections.singletonList(new byte[] { 1, 2, 3, 4 });
    List<List<byte[]>> list2 = Collections.singletonList(innerList2);
    Row row2 = Row.withSchema(schema).addValue(list2).build();
    Assume.assumeTrue(coder.consistentWithEquals());
    CoderProperties.coderConsistentWithEquals(coder, row1, row2);
}

Example 48 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testArrayOfArray.

@Test
public void testArrayOfArray() throws Exception {
    FieldType arrayType = FieldType.array(FieldType.array(FieldType.INT32));
    Schema schema = Schema.builder().addField("f_array", arrayType).build();
    Row row = Row.withSchema(schema).addArray(Lists.newArrayList(1, 2, 3, 4), Lists.newArrayList(5, 6, 7, 8), Lists.newArrayList(9, 10, 11, 12)).build();
    CoderProperties.coderDecodeEncodeEqual(RowCoder.of(schema), row);
}

Example 49 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class RowCoderTest method testEncodingPositionAddNewFields.

@Test
public void testEncodingPositionAddNewFields() throws Exception {
    Schema schema1 = Schema.builder().addNullableField("f_int32", FieldType.INT32).addNullableField("f_string", FieldType.STRING).build();
    Schema schema2 = Schema.builder().addNullableField("f_int32", FieldType.INT32).addNullableField("f_string", FieldType.STRING).addNullableField("f_boolean", FieldType.BOOLEAN).build();
    Row row = Row.withSchema(schema1).withFieldValue("f_int32", 42).withFieldValue("f_string", "hello world!").build();
    Row expected = Row.withSchema(schema2).withFieldValue("f_int32", 42).withFieldValue("f_string", "hello world!").build();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    RowCoder.of(schema1).encode(row, os);
    Row decoded = RowCoder.of(schema2).decode(new ByteArrayInputStream(os.toByteArray()));
    assertEquals(expected, decoded);
}

Example 50 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class BeamSqlPojoExample method main.

public static void main(String[] args) {
    PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create();
    Pipeline pipeline = Pipeline.create(options);
    // First step is to get PCollections of source objects.
    // In this example we create them directly in memory using Create.of().
    // 
    // In real world such PCollections will likely be obtained from some other source,
    // e.g. a database or a text file. This process is not specific to Beam SQL,
    // please consult Beam programming guide for details.
    PCollection<Customer> customers = loadCustomers(pipeline);
    PCollection<Order> orders = loadOrders(pipeline);
    // Example 1. Run a simple query over java objects:
    PCollection<Row> customersFromWonderland = customers.apply(SqlTransform.query("SELECT id, name " + " FROM PCOLLECTION " + " WHERE countryOfResidence = 'Wonderland'"));
    // Output the results of the query:
    customersFromWonderland.apply(logRecords(": is from Wonderland"));
    // Example 2. Query the results of the first query:
    PCollection<Row> totalInWonderland = customersFromWonderland.apply(SqlTransform.query("SELECT COUNT(id) FROM PCOLLECTION"));
    // Output the results of the query:
    totalInWonderland.apply(logRecords(": total customers in Wonderland"));
    // Example 3. Query multiple PCollections of Java objects:
    PCollection<Row> ordersByGrault = PCollectionTuple.of(new TupleTag<>("customers"), customers).and(new TupleTag<>("orders"), orders).apply(SqlTransform.query("SELECT customers.name, ('order id:' || CAST(orders.id AS VARCHAR))" + " FROM orders " + "   JOIN customers ON orders.customerId = customers.id" + " WHERE customers.name = 'Grault'"));
    // Output the results of the query:
    ordersByGrault.apply(logRecords(": ordered by 'Grault'"));
    pipeline.run().waitUntilFinish();
}