use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project drill by apache.
the class FilePushDownFilter method doOnMatch.
protected void doOnMatch(RelOptRuleCall call, FilterPrel filter, ProjectPrel project, ScanPrel scan) {
AbstractGroupScanWithMetadata<?> groupScan = (AbstractGroupScanWithMetadata<?>) scan.getGroupScan();
if (groupScan.getFilter() != null && !groupScan.getFilter().equals(ValueExpressions.BooleanExpression.TRUE)) {
return;
}
RexNode condition;
if (project == null) {
condition = filter.getCondition();
} else {
// get the filter as if it were below the projection.
condition = RelOptUtil.pushPastProject(filter.getCondition(), project);
}
if (condition == null || condition.isAlwaysTrue()) {
return;
}
// get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
// then we could not pushed down. Otherwise, it's qualified to be pushed down.
// Limits the number of nodes that can be created out of the conversion to avoid
// exponential growth of nodes count and further OOM
final List<RexNode> predList = RelOptUtil.conjunctions(RexUtil.toCnf(filter.getCluster().getRexBuilder(), 100, condition));
final List<RexNode> qualifiedPredList = new ArrayList<>();
// list of predicates which cannot be converted to filter predicate
List<RexNode> nonConvertedPredList = new ArrayList<>();
for (RexNode pred : predList) {
if (DrillRelOptUtil.findOperators(pred, Collections.emptyList(), BANNED_OPERATORS) == null) {
LogicalExpression drillPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, pred);
// checks whether predicate may be used for filter pushdown
FilterPredicate<?> filterPredicate = groupScan.getFilterPredicate(drillPredicate, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions(), false);
// to build filter with them
if (filterPredicate == null) {
nonConvertedPredList.add(pred);
}
qualifiedPredList.add(pred);
} else {
nonConvertedPredList.add(pred);
}
}
final RexNode qualifiedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
if (qualifiedPred == null) {
return;
}
LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifiedPred);
// Default - pass the original filter expr to (potentially) be used at run-time
// later may remove or set to another filter (see below)
groupScan.setFilterForRuntime(conditionExp, optimizerContext);
Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
AbstractGroupScanWithMetadata<?> newGroupScan = groupScan.applyFilter(conditionExp, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions());
if (timer != null) {
logger.debug("Took {} ms to apply filter. ", timer.elapsed(TimeUnit.MILLISECONDS));
timer.stop();
}
// fully match the filter for the case when row group pruning did not happen.
if (newGroupScan == null) {
if (groupScan.isMatchAllMetadata()) {
RelNode child = project == null ? scan : project;
// but row group pruning did not happen, remove the filter.
if (nonConvertedPredList.isEmpty()) {
// disable the original filter expr (i.e. don't use it at run-time)
groupScan.setFilterForRuntime(null, optimizerContext);
call.transformTo(child);
} else if (nonConvertedPredList.size() == predList.size()) {
// None of the predicates participated in filter pushdown.
return;
} else {
// If some of the predicates weren't used in the filter, creates new filter with them
// on top of current scan. Excludes the case when all predicates weren't used in the filter.
Filter theNewFilter = filter.copy(filter.getTraitSet(), child, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theNewFilter.getCondition());
// pass the new filter expr to (potentialy) be used at run-time
groupScan.setFilterForRuntime(filterPredicate, optimizerContext);
// Replace the child with the new filter on top of the child/scan
call.transformTo(theNewFilter);
}
}
return;
}
RelNode newNode = new ScanPrel(scan.getCluster(), scan.getTraitSet(), newGroupScan, scan.getRowType(), scan.getTable());
if (project != null) {
newNode = project.copy(project.getTraitSet(), Collections.singletonList(newNode));
}
if (newGroupScan.isMatchAllMetadata()) {
// creates filter from the expressions which can't be pushed to the scan
if (!nonConvertedPredList.isEmpty()) {
Filter theFilterRel = filter.copy(filter.getTraitSet(), newNode, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theFilterRel.getCondition());
// pass the new filter expr to (potentialy) be used at run-time
newGroupScan.setFilterForRuntime(filterPredicate, optimizerContext);
// replace the new node with the new filter on top of that new node
newNode = theFilterRel;
}
call.transformTo(newNode);
return;
}
final RelNode newFilter = filter.copy(filter.getTraitSet(), Collections.singletonList(newNode));
call.transformTo(newFilter);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project drill by apache.
the class DrillFilterAggregateTransposeRule method matches.
@Override
public boolean matches(RelOptRuleCall call) {
final Filter filter = call.rel(0);
final Aggregate aggregate = call.rel(1);
return filter.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == aggregate.getTraitSet().getTrait(ConventionTraitDef.INSTANCE);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project drill by apache.
the class ElasticsearchFilterRule method convert.
@Override
public RelNode convert(RelNode relNode) {
Filter filter = (Filter) relNode;
NodeTypeFinder filterFinder = new NodeTypeFinder(ElasticsearchFilter.class);
filter.getInput().accept(filterFinder);
if (filterFinder.containsNode) {
return null;
}
RelTraitSet traitSet = filter.getTraitSet().replace(out);
try {
CalciteUtils.analyzePredicate(filter.getCondition());
} catch (Exception e) {
logger.info("Unable to push filter into ElasticSearch :{}", e.getMessage(), e);
return null;
}
return CalciteUtils.createFilter(traitSet, convert(filter.getInput(), out), filter.getCondition());
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project flink by apache.
the class PushPartitionIntoTableSourceScanRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
Filter filter = call.rel(0);
LogicalTableScan scan = call.rel(1);
TableSourceTable tableSourceTable = scan.getTable().unwrap(TableSourceTable.class);
RelDataType inputFieldTypes = filter.getInput().getRowType();
List<String> inputFieldNames = inputFieldTypes.getFieldNames();
List<String> partitionFieldNames = tableSourceTable.contextResolvedTable().<ResolvedCatalogTable>getResolvedTable().getPartitionKeys();
// extract partition predicates
RelBuilder relBuilder = call.builder();
RexBuilder rexBuilder = relBuilder.getRexBuilder();
Tuple2<Seq<RexNode>, Seq<RexNode>> allPredicates = RexNodeExtractor.extractPartitionPredicateList(filter.getCondition(), FlinkRelOptUtil.getMaxCnfNodeCount(scan), inputFieldNames.toArray(new String[0]), rexBuilder, partitionFieldNames.toArray(new String[0]));
RexNode partitionPredicate = RexUtil.composeConjunction(rexBuilder, JavaConversions.seqAsJavaList(allPredicates._1));
if (partitionPredicate.isAlwaysTrue()) {
return;
}
// build pruner
LogicalType[] partitionFieldTypes = partitionFieldNames.stream().map(name -> {
int index = inputFieldNames.indexOf(name);
if (index < 0) {
throw new TableException(String.format("Partitioned key '%s' isn't found in input columns. " + "Validator should have checked that.", name));
}
return inputFieldTypes.getFieldList().get(index).getType();
}).map(FlinkTypeFactory::toLogicalType).toArray(LogicalType[]::new);
RexNode finalPartitionPredicate = adjustPartitionPredicate(inputFieldNames, partitionFieldNames, partitionPredicate);
FlinkContext context = ShortcutUtils.unwrapContext(scan);
Function<List<Map<String, String>>, List<Map<String, String>>> defaultPruner = partitions -> PartitionPruner.prunePartitions(context.getTableConfig(), partitionFieldNames.toArray(new String[0]), partitionFieldTypes, partitions, finalPartitionPredicate);
// prune partitions
List<Map<String, String>> remainingPartitions = readPartitionsAndPrune(rexBuilder, context, tableSourceTable, defaultPruner, allPredicates._1(), inputFieldNames);
// apply push down
DynamicTableSource dynamicTableSource = tableSourceTable.tableSource().copy();
PartitionPushDownSpec partitionPushDownSpec = new PartitionPushDownSpec(remainingPartitions);
partitionPushDownSpec.apply(dynamicTableSource, SourceAbilityContext.from(scan));
// build new statistic
TableStats newTableStat = null;
if (tableSourceTable.contextResolvedTable().isPermanent()) {
ObjectIdentifier identifier = tableSourceTable.contextResolvedTable().getIdentifier();
ObjectPath tablePath = identifier.toObjectPath();
Catalog catalog = tableSourceTable.contextResolvedTable().getCatalog().get();
for (Map<String, String> partition : remainingPartitions) {
Optional<TableStats> partitionStats = getPartitionStats(catalog, tablePath, partition);
if (!partitionStats.isPresent()) {
// clear all information before
newTableStat = null;
break;
} else {
newTableStat = newTableStat == null ? partitionStats.get() : newTableStat.merge(partitionStats.get());
}
}
}
FlinkStatistic newStatistic = FlinkStatistic.builder().statistic(tableSourceTable.getStatistic()).tableStats(newTableStat).build();
TableSourceTable newTableSourceTable = tableSourceTable.copy(dynamicTableSource, newStatistic, new SourceAbilitySpec[] { partitionPushDownSpec });
LogicalTableScan newScan = LogicalTableScan.create(scan.getCluster(), newTableSourceTable, scan.getHints());
// transform to new node
RexNode nonPartitionPredicate = RexUtil.composeConjunction(rexBuilder, JavaConversions.seqAsJavaList(allPredicates._2()));
if (nonPartitionPredicate.isAlwaysTrue()) {
call.transformTo(newScan);
} else {
Filter newFilter = filter.copy(filter.getTraitSet(), newScan, nonPartitionPredicate);
call.transformTo(newFilter);
}
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project flink by apache.
the class PushPartitionIntoTableSourceScanRule method matches.
@Override
public boolean matches(RelOptRuleCall call) {
Filter filter = call.rel(0);
if (filter.getCondition() == null) {
return false;
}
TableSourceTable tableSourceTable = call.rel(1).getTable().unwrap(TableSourceTable.class);
if (tableSourceTable == null) {
return false;
}
DynamicTableSource dynamicTableSource = tableSourceTable.tableSource();
if (!(dynamicTableSource instanceof SupportsPartitionPushDown)) {
return false;
}
CatalogTable catalogTable = tableSourceTable.contextResolvedTable().getTable();
if (!catalogTable.isPartitioned() || catalogTable.getPartitionKeys().isEmpty()) {
return false;
}
return Arrays.stream(tableSourceTable.abilitySpecs()).noneMatch(spec -> spec instanceof PartitionPushDownSpec);
}
Aggregations