Examples with BeamRelNode org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode used on opensource projects

Example 61 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterAndProjectPushDown method testIOSourceRel_withUnsupportedPredicate.

@Test
public void testIOSourceRel_withUnsupportedPredicate() {
    String selectTableStatement = "SELECT name FROM TEST where id+unused1=101";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    assertEquals("BeamPushDownIOSourceRel.BEAM_LOGICAL(table=[beam, TEST],usedFields=[name, id, unused1],TestTableFilter=[supported{}, unsupported{=(+($1, $0), 101)}])", beamRelNode.getInput(0).getDigest());
    // Make sure project push-down was done
    List<String> a = beamRelNode.getInput(0).getRowType().getFieldNames();
    assertThat(a, containsInAnyOrder("name", "id", "unused1"));
    assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "one"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}

Example 62 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterAndProjectPushDown method testIOSourceRel_predicateSimple_Boolean.

@Test
public void testIOSourceRel_predicateSimple_Boolean() {
    String selectTableStatement = "SELECT name FROM TEST where b";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamIOSourceRel.class));
    assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "one"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}

Example 63 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithProjectPushDown method testIOSourceRel_withPredicate_withRename.

@Test
public void testIOSourceRel_withPredicate_withRename() {
    String selectTableStatement = "SELECT name as new_name FROM TEST where id=2";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertEquals(result.getSchema(), Schema.builder().addStringField("new_name").build());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "two"));
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    // When doing only project push-down, predicate (and rename) should be preserved in a Calc
    assertThat(beamRelNode.getRowType().getFieldNames(), containsInAnyOrder("new_name"));
    // IO should project fields queried + fields used by the predicate
    assertThat(beamRelNode.getInput(0).getRowType().getFieldNames(), containsInAnyOrder("id", "name"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}

Example 64 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class ZetaSqlTimeFunctionsTest method testTimeFromTimestamp.

@Test
public void testTimeFromTimestamp() {
    String sql = "SELECT TIME(TIMESTAMP '2008-12-25 15:30:00+08', 'America/Los_Angeles')";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(Schema.builder().addLogicalTypeField("f_time", SqlTypes.TIME).build()).addValues(LocalTime.of(23, 30, 0)).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}

Example 65 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class ZetaSqlTimeFunctionsTest method testTimestampDiff.

@Test
public void testTimestampDiff() {
    String sql = "SELECT TIMESTAMP_DIFF(" + "TIMESTAMP '2018-10-14 15:30:00.000 UTC', " + "TIMESTAMP '2018-08-14 15:05:00.001 UTC', " + "MILLISECOND)";
    ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
    BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
    PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    PAssert.that(stream).containsInAnyOrder(Row.withSchema(Schema.builder().addInt64Field("f_timestamp_diff").build()).addValues((61L * 24 * 60 + 25) * 60 * 1000 - 1).build());
    pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
}