Examples with Row org.apache.beam.sdk.values.Row used on opensource projects

Example 86 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class StreamingSqlTest method runSessionWindow2.

@Test
public void runSessionWindow2() throws Exception {
    String sql = "SELECT f_long, f_string, COUNT(*) AS `getFieldCount`," + " `window_start`, `window_end` " + " FROM SESSION((select * from streaming_sql_test_table_a), descriptor(f_timestamp), " + " descriptor(f_long, f_string), \"INTERVAL 5 MINUTE\")" + " GROUP BY f_long, f_string, window_start, window_end";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    Schema resultType = Schema.builder().addInt64Field("f_long").addStringField("f_string").addInt64Field("size").addDateTimeField("window_start").addDateTimeField("window_end").build();
    List<Row> expectedRows = Arrays.asList(Row.withSchema(resultType).addValues(1000L, "string_row1", 2L, parseTimestampWithUTCTimeZone("2017-01-01 01:01:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:07:03")).build(), Row.withSchema(resultType).addValues(1000L, "string_row3", 1L, parseTimestampWithUTCTimeZone("2017-01-01 01:06:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:11:03")).build(), Row.withSchema(resultType).addValues(4000L, "第四行", 1L, parseTimestampWithUTCTimeZone("2017-01-01 02:04:03"), parseTimestampWithUTCTimeZone("2017-01-01 02:09:03")).build());
    PAssert.that(stream).containsInAnyOrder(expectedRows);
    pipeline.run().waitUntilFinish();
}

Example 87 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class StreamingSqlTest method runSessionWindow.

@Test
public void runSessionWindow() throws Exception {
    String sql = "SELECT f_long, COUNT(*) AS `getFieldCount`," + " `window_start`, " + " `window_end` " + " FROM SESSION((select * from streaming_sql_test_table_a), descriptor(f_timestamp), " + " descriptor(f_long), \"INTERVAL 5 MINUTE\")" + " GROUP BY f_long, window_start, window_end";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    Schema resultType = Schema.builder().addInt64Field("f_long").addInt64Field("size").addDateTimeField("window_start").addDateTimeField("window_end").build();
    List<Row> expectedRows = Arrays.asList(Row.withSchema(resultType).addValues(1000L, 3L, parseTimestampWithUTCTimeZone("2017-01-01 01:01:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:11:03")).build(), Row.withSchema(resultType).addValues(4000L, 1L, parseTimestampWithUTCTimeZone("2017-01-01 02:04:03"), parseTimestampWithUTCTimeZone("2017-01-01 02:09:03")).build());
    PAssert.that(stream).containsInAnyOrder(expectedRows);
    pipeline.run().waitUntilFinish();
}

Example 88 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class StreamingSqlTest method testWithQuerySeven.

@Test
public void testWithQuerySeven() {
    String sql = "WITH T1 AS (SELECT * FROM KeyValue) SELECT " + "COUNT(*) as field_count, " + "TUMBLE_START(\"INTERVAL 1 SECOND\") as window_start, " + "TUMBLE_END(\"INTERVAL 1 SECOND\") as window_end " + "FROM T1 " + "GROUP BY TUMBLE(ts, \"INTERVAL 1 SECOND\");";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    final Schema schema = Schema.builder().addInt64Field("count_start").addDateTimeField("field1").addDateTimeField("field2").build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC())).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 89 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class ZetaSqlDialectSpecTest method testLike1.

@Test
public void testLike1() {
    String sql = "SELECT @p0 LIKE @p1 AS ColA";
    ImmutableMap<String, Value> params = ImmutableMap.of("p0", Value.createStringValue("ab%"), "p1", Value.createStringValue("ab\\%"));
    PCollection<Row> stream = execute(sql, params);
    final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 90 with Row

use of org.apache.beam.sdk.values.Row in project beam by apache.

the class ZetaSqlDialectSpecTest method testZetaSQLRightOuterJoin.

@Test
public void testZetaSQLRightOuterJoin() {
    String sql = "SELECT * " + "FROM KeyValue AS t1" + " RIGHT JOIN BigTable AS t2" + " on " + " t1.Key = t2.RowKey";
    PCollection<Row> stream = execute(sql);
    final Schema schemaOne = Schema.builder().addNullableField("field1", FieldType.INT64).addNullableField("field2", FieldType.STRING).addNullableField("field3", DATETIME).addInt64Field("field4").addStringField("field5").addDateTimeField("field6").build();
    final Schema schemaTwo = Schema.builder().addInt64Field("field1").addStringField("field2").addDateTimeField("field3").addInt64Field("field4").addStringField("field5").addDateTimeField("field6").build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(schemaOne).addValues(null, null, null, 16L, "BigTable236", new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schemaTwo).addValues(15L, "KeyValue235", new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()), 15L, "BigTable235", new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC())).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}