use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.
the class TestTableProviderWithFilterPushDown method testIOSourceRel_withFilter_shouldProjectAllFields.
@Test
public void testIOSourceRel_withFilter_shouldProjectAllFields() {
String selectTableStatement = "SELECT name FROM TEST where name='two'";
BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
// Condition should be pushed-down to IO level
assertNull(((Calc) beamRelNode).getProgram().getCondition());
assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
// When performing standalone filter push-down IO should project all fields.
assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "two"));
pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.
the class TestTableProviderWithFilterPushDown method testIOSourceRel_withUnsupportedFilter_calcPreservesCondition.
@Test
public void testIOSourceRel_withUnsupportedFilter_calcPreservesCondition() {
String selectTableStatement = "SELECT name FROM TEST where id+1=2";
BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
// Unsupported condition should be preserved in a Calc
assertNotNull(((Calc) beamRelNode).getProgram().getCondition());
assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
// When performing standalone filter push-down IO should project all fields.
assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
assertEquals(Schema.builder().addStringField("name").build(), result.getSchema());
PAssert.that(result).containsInAnyOrder(row(result.getSchema(), "one"));
pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.
the class TestTableProviderWithFilterPushDown method testIOSourceRel_withSupportedFilter_selectInRandomOrder.
@Test
public void testIOSourceRel_withSupportedFilter_selectInRandomOrder() {
String selectTableStatement = "SELECT unused2, id, name FROM TEST where b";
BeamRelNode beamRelNode = sqlEnv.parseQuery(selectTableStatement);
PCollection<Row> result = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
assertThat(beamRelNode, instanceOf(BeamCalcRel.class));
// Condition should be pushed-down to IO level
assertNull(((Calc) beamRelNode).getProgram().getCondition());
assertThat(beamRelNode.getInput(0), instanceOf(BeamIOSourceRel.class));
List<String> projects = beamRelNode.getInput(0).getRowType().getFieldNames();
// When performing standalone filter push-down IO should project all fields.
assertThat(projects, containsInAnyOrder("unused1", "id", "name", "unused2", "b"));
assertEquals(Schema.builder().addInt16Field("unused2").addInt32Field("id").addStringField("name").build(), result.getSchema());
PAssert.that(result).containsInAnyOrder(row(result.getSchema(), (short) 100, 1, "one"));
pipeline.run().waitUntilFinish(Duration.standardMinutes(2));
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.
the class BeamBasicAggregationRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
Aggregate aggregate = call.rel(0);
RelNode relNode = call.rel(1);
if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
return;
}
if (relNode instanceof Project || relNode instanceof Calc || relNode instanceof Filter) {
if (isWindowed(relNode) || hasWindowedParents(relNode)) {
// This case is expected to get handled by the 'BeamAggregationRule'
return;
}
}
RelNode newTableScan = relNode.copy(relNode.getTraitSet(), relNode.getInputs());
call.transformTo(new BeamAggregationRel(aggregate.getCluster(), aggregate.getTraitSet().replace(BeamLogicalConvention.INSTANCE), convert(newTableScan, newTableScan.getTraitSet().replace(BeamLogicalConvention.INSTANCE)), aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList(), null, -1));
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.
the class BeamIOPushDownRule method constructNodesWithPushDown.
/**
* Construct a new {@link BeamIOSourceRel} with predicate and/or project pushed-down and a new
* {@code Calc} to do field reordering/field duplication/complex projects.
*
* @param resolved A descriptor of fields used by a {@code Calc}.
* @param relBuilder A {@code RelBuilder} for constructing {@code Project} and {@code Filter} Rel
* nodes with operations unsupported by the IO.
* @param ioSourceRel Original {@code BeamIOSourceRel} we are attempting to perform push-down for.
* @param tableFilter A class containing information about IO predicate push-down capabilities.
* @param calcDataType A Calcite output schema of an original {@code Calc}.
* @param calcProjects A list of projected {@code RexNode}s by a {@code Calc}.
* @return An alternative {@code RelNode} with supported filters/projects pushed-down to IO Rel.
*/
private RelNode constructNodesWithPushDown(FieldAccessDescriptor resolved, RelBuilder relBuilder, BeamIOSourceRel ioSourceRel, BeamSqlTableFilter tableFilter, RelDataType calcDataType, List<RexNode> calcProjects) {
Schema newSchema = SelectHelpers.getOutputSchema(ioSourceRel.getBeamSqlTable().getSchema(), resolved);
RelDataType calcInputType = CalciteUtils.toCalciteRowType(newSchema, ioSourceRel.getCluster().getTypeFactory());
BeamIOSourceRel newIoSourceRel = ioSourceRel.createPushDownRel(calcInputType, newSchema.getFieldNames(), tableFilter);
relBuilder.push(newIoSourceRel);
List<RexNode> newProjects = new ArrayList<>();
List<RexNode> newFilter = new ArrayList<>();
// Ex: let's say the original fields are (number before each element is the index):
// {0:unused1, 1:id, 2:name, 3:unused2},
// where only 'id' and 'name' are being used. Then the new calcInputType should be as follows:
// {0:id, 1:name}.
// A mapping list will contain 2 entries: {0:1, 1:2},
// showing how used field names map to the original fields.
List<Integer> mapping = resolved.getFieldsAccessed().stream().map(FieldDescriptor::getFieldId).collect(Collectors.toList());
// Map filters to new RexInputRef.
for (RexNode filter : tableFilter.getNotSupported()) {
newFilter.add(reMapRexNodeToNewInputs(filter, mapping));
}
// Map projects to new RexInputRef.
for (RexNode project : calcProjects) {
newProjects.add(reMapRexNodeToNewInputs(project, mapping));
}
if (RexUtil.isIdentity(newProjects, newIoSourceRel.getRowType())) {
// Force a rename prior to filter for identity function.
relBuilder.project(newProjects, calcDataType.getFieldNames(), true);
}
relBuilder.filter(newFilter);
relBuilder.project(newProjects, calcDataType.getFieldNames());
return relBuilder.build();
}
Aggregations