Examples with BeamRelNode org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode used on opensource projects

Example 26 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class StreamingSqlTest method testZetaSQLBasicSlidingWindowing.

@Test
public void testZetaSQLBasicSlidingWindowing() {
    String sql = "SELECT " + "COUNT(*) as field_count, " + "HOP_START(\"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\") as window_start, " + "HOP_END(\"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\") as window_end " + "FROM window_test_table " + "GROUP BY HOP(ts, \"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\");";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    final Schema schema = Schema.builder().addInt64Field("count_star").addDateTimeField("field1").addDateTimeField("field2").build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(2L, new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 9, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 5, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(2L, new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(2L, new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 10, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 9, ISOChronology.getInstanceUTC()), new DateTime(2018, 7, 1, 21, 26, 11, ISOChronology.getInstanceUTC())).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 27 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class StreamingSqlTest method testZetaSQLNestedQueryFour.

@Test
public void testZetaSQLNestedQueryFour() {
    String sql = "SELECT t1.Value, TUMBLE_START('INTERVAL 1 SECOND') AS period_start, MIN(t2.Value) as" + " min_v FROM KeyValue AS t1 INNER JOIN BigTable AS t2 on t1.Key = t2.RowKey GROUP BY" + " t1.Value, TUMBLE(t2.ts, 'INTERVAL 1 SECOND')";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(Schema.builder().addStringField("value").addDateTimeField("min_v").addStringField("period_start").build()).addValues("KeyValue235", new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()), "BigTable235").build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 28 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class StreamingSqlTest method testTVFTumbleAggregation.

@Test
public void testTVFTumbleAggregation() {
    String sql = "SELECT COUNT(*) as field_count, " + "window_start " + "FROM TUMBLE((select * from KeyValue), descriptor(ts), 'INTERVAL 1 SECOND') " + "GROUP BY window_start";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    final Schema schema = Schema.builder().addInt64Field("field_count").addDateTimeField("window_start").build();
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC())).build(), Row.withSchema(schema).addValues(1L, new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC())).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 29 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class StreamingSqlTest method runSessionWindow2.

@Test
public void runSessionWindow2() throws Exception {
    String sql = "SELECT f_long, f_string, COUNT(*) AS `getFieldCount`," + " `window_start`, `window_end` " + " FROM SESSION((select * from streaming_sql_test_table_a), descriptor(f_timestamp), " + " descriptor(f_long, f_string), \"INTERVAL 5 MINUTE\")" + " GROUP BY f_long, f_string, window_start, window_end";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    Schema resultType = Schema.builder().addInt64Field("f_long").addStringField("f_string").addInt64Field("size").addDateTimeField("window_start").addDateTimeField("window_end").build();
    List<Row> expectedRows = Arrays.asList(Row.withSchema(resultType).addValues(1000L, "string_row1", 2L, parseTimestampWithUTCTimeZone("2017-01-01 01:01:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:07:03")).build(), Row.withSchema(resultType).addValues(1000L, "string_row3", 1L, parseTimestampWithUTCTimeZone("2017-01-01 01:06:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:11:03")).build(), Row.withSchema(resultType).addValues(4000L, "第四行", 1L, parseTimestampWithUTCTimeZone("2017-01-01 02:04:03"), parseTimestampWithUTCTimeZone("2017-01-01 02:09:03")).build());
    PAssert.that(stream).containsInAnyOrder(expectedRows);
    pipeline.run().waitUntilFinish();
}

Example 30 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class StreamingSqlTest method runSessionWindow.

@Test
public void runSessionWindow() throws Exception {
    String sql = "SELECT f_long, COUNT(*) AS `getFieldCount`," + " `window_start`, " + " `window_end` " + " FROM SESSION((select * from streaming_sql_test_table_a), descriptor(f_timestamp), " + " descriptor(f_long), \"INTERVAL 5 MINUTE\")" + " GROUP BY f_long, window_start, window_end";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    Schema resultType = Schema.builder().addInt64Field("f_long").addInt64Field("size").addDateTimeField("window_start").addDateTimeField("window_end").build();
    List<Row> expectedRows = Arrays.asList(Row.withSchema(resultType).addValues(1000L, 3L, parseTimestampWithUTCTimeZone("2017-01-01 01:01:03"), parseTimestampWithUTCTimeZone("2017-01-01 01:11:03")).build(), Row.withSchema(resultType).addValues(4000L, 1L, parseTimestampWithUTCTimeZone("2017-01-01 02:04:03"), parseTimestampWithUTCTimeZone("2017-01-01 02:09:03")).build());
    PAssert.that(stream).containsInAnyOrder(expectedRows);
    pipeline.run().waitUntilFinish();
}