Examples with BeamCalcRel org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel used on opensource projects

Example 1 with BeamCalcRel

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_selectAllFieldsInRandomOrder_shouldPushDownSupportedFilter.

@Test
public void testIOSourceRel_selectAllFieldsInRandomOrder_shouldPushDownSupportedFilter() {
    String selectTableStatement = "SELECT unused2, name, id, b, unused1 FROM TEST where name='two'";
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    // Calc should not be dropped, because fields are selected in a different order, even though
    // all filters are supported and all fields are projected.
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    assertNull(((BeamCalcRel) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
    // When performing standalone filter push-down IO should project all fields.
    assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addInt16Field("unused2").addStringField("name").addInt32Field("id").addBooleanField("b").addInt32Field("unused1").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), (short) 200, "two", 2, false, 200));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}

Example 2 with BeamCalcRel

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel in project beam by apache.

the class TestTableProviderWithFilterPushDown method testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate.

@Test
public void testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate() {
    String selectTableStatement = "SELECT b, b, b, b, b FROM TEST where b";
    // Calc must not be dropped
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Supported predicate should be pushed-down
    assertNull(((BeamCalcRel) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    // Make sure project push-down was done
    List<String> pushedFields = beamRelNode.getInput(0).getRowType().getFieldNames();
    assertThat(pushedFields, IsIterableContainingInAnyOrder.containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
    assertEquals(Schema.builder().addBooleanField("b").addBooleanField("b0").addBooleanField("b1").addBooleanField("b2").addBooleanField("b3").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), true, true, true, true, true));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}

Example 3 with BeamCalcRel

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel in project beam by apache.

the class MongoDbFilterTest method testIsSupported.

@Test
public void testIsSupported() {
    BeamRelNode beamRelNode = sqlEnv.parseQuery(query);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    MongoDbFilter filter = MongoDbFilter.create(((BeamCalcRel) beamRelNode).getProgram().split().right);
    assertThat("Query: '" + query + "' is expected to be " + (isSupported ? "supported." : "unsupported."), filter.getNotSupported().isEmpty() == isSupported);
}

Example 4 with BeamCalcRel

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel in project beam by apache.

the class BeamCalcRule method convert.

@Override
public RelNode convert(RelNode rel) {
    final Calc calc = (Calc) rel;
    final RelNode input = calc.getInput();
    return new BeamCalcRel(calc.getCluster(), calc.getTraitSet().replace(BeamLogicalConvention.INSTANCE), RelOptRule.convert(input, input.getTraitSet().replace(BeamLogicalConvention.INSTANCE)), calc.getProgram());
}

Example 5 with BeamCalcRel

use of org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel in project beam by apache.

the class TestTableProviderWithProjectPushDown method testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate.

@Test
public void testIOSourceRel_selectOneFieldsMoreThanOnce_withSupportedPredicate() {
    String selectTableStatement = "SELECT id, id, id, id, id FROM TEST where id=1";
    // Calc must not be dropped
    BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
    PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
    assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
    // Project push-down should leave predicate in a Calc
    assertNotNull(((BeamCalcRel) beamRelNode).getProgram().getCondition());
    assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
    // Make sure project push-down was done
    List<String> pushedFields = beamRelNode.getInput(0).getRowType().getFieldNames();
    assertThat(pushedFields, containsInAnyOrder("id"));
    assertEquals(Schema.builder().addInt32Field("id").addInt32Field("id0").addInt32Field("id1").addInt32Field("id2").addInt32Field("id3").build(), result.getSchema());
    PAssert.that(result).containsInAnyOrder(row(result.getSchema(), 1, 1, 1, 1, 1));
    pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}