Search in sources :

Example 51 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_withFilter_shouldProjectAllFields.

@Test
public void testIOSourceRel_withFilter_shouldProjectAllFields() {
    String selectTableStatement = "SELECT name FROM TEST where name='two'";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Condition should be pushed-down to IO level
    assertNull(((Calc) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
    // When performing standalone filter push-down IO should project all fields.
    assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "two"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
Also used : BeamCalcRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamIOSourceRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel) Calc(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc) Row(org.apache.beam.sdk.values.Row) Test(org.junit.Test)

Example 52 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_withUnsupportedFilter_calcPreservesCondition.

@Test
public void testIOSourceRel_withUnsupportedFilter_calcPreservesCondition() {
    String selectTableStatement = "SELECT name FROM TEST where id+1=2";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Unsupported condition should be preserved in a Calc
    assertNotNull(((Calc) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
    // When performing standalone filter push-down IO should project all fields.
    assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "one"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
Also used : BeamCalcRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamIOSourceRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel) Calc(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc) Row(org.apache.beam.sdk.values.Row) Test(org.junit.Test)

Example 53 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_withSupportedFilter_selectInRandomOrder.

@Test
public void testIOSourceRel_withSupportedFilter_selectInRandomOrder() {
    String selectTableStatement = "SELECT unused2, id, name FROM TEST where b";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Condition should be pushed-down to IO level
    assertNull(((Calc) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
    // When performing standalone filter push-down IO should project all fields.
    assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addInt16Field("unused2").addInt32Field("id").addStringField("name").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), (short) 100, 1, "one"));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
Also used : BeamCalcRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamIOSourceRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel) Calc(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc) Row(org.apache.beam.sdk.values.Row) Test(org.junit.Test)

Example 54 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate.

@Test
public void testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate() {
    String selectTableStatement = "SELECT b, b, b, b, b FROM TEST where b";
    // Calc must not be dropped
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Supported predicate should be pushed-down
    assertNull(((BeamCalcRel) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    // Make sure project push-down was done
    List<String> pushedFields = beamRelNode.getInput(0).getRowType().getFieldNames();
    assertThat(pushedFields, IsIterableContainingInAnyOrder.containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addBooleanField("b").addBooleanField("b0").addBooleanField("b1").addBooleanField("b2").addBooleanField("b3").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), true, true, true, true, true));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
Also used : BeamCalcRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamIOSourceRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel) Row(org.apache.beam.sdk.values.Row) Test(org.junit.Test)

Example 55 with BeamRelNode

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode in project beam by apache.

the class MongoDbFilterTest method testIsSupported.

@Test
public void testIsSupported() {
    BeamRelNode beamRelNode = sqlEnv.parseQuery(query);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    MongoDbFilter filter = MongoDbFilter.create(((BeamCalcRel) beamRelNode).getProgram().split().right);
    assertThat("Query: '" + query + "' is expected to be " + (isSupported ? "supported." : "unsupported."), filter.getNotSupported().isEmpty() == isSupported);
}
Also used : BeamCalcRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) MongoDbFilter(org.apache.beam.sdk.extensions.sql.meta.provider.mongodb.MongoDbTable.MongoDbFilter) Test(org.junit.Test)

Aggregations

BeamRelNode (org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode)246 Test (org.junit.Test)241 Row (org.apache.beam.sdk.values.Row)207 Schema (org.apache.beam.sdk.schemas.Schema)54 BeamIOSourceRel (org.apache.beam.sdk.extensions.sql.impl.rel.BeamIOSourceRel)38 ByteString (com.google.protobuf.ByteString)24 BeamCalcRel (org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel)22 Matchers.containsString (org.hamcrest.Matchers.containsString)13 Value (com.google.zetasql.Value)9 DateTime (org.joda.time.DateTime)8 BeamSqlEnv (org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv)7 BeamPushDownIOSourceRel (org.apache.beam.sdk.extensions.sql.impl.rel.BeamPushDownIOSourceRel)6 Calc (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Calc)6 PipelineResult (org.apache.beam.sdk.PipelineResult)5 TestTableProvider (org.apache.beam.sdk.extensions.sql.meta.provider.test.TestTableProvider)5 DateTimeUtils.parseDateToValue (org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseDateToValue)5 DateTimeUtils.parseTimeToValue (org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimeToValue)5 DateTimeUtils.parseTimestampWithTZToValue (org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithTZToValue)5 Ignore (org.junit.Ignore)5 BeamAggregationRel (org.apache.beam.sdk.extensions.sql.impl.rel.BeamAggregationRel)4