use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode in project druid by druid-io.
the class QuantileSqlAggregator method toDruidAggregation.
@Override
public Aggregation toDruidAggregation(final String name, final RowSignature rowSignature, final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final List<Aggregation> existingAggregations, final Project project, final AggregateCall aggregateCall, final DimFilter filter) {
final RowExtraction rex = Expressions.toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(0)));
if (rex == null) {
return null;
}
final AggregatorFactory aggregatorFactory;
final String histogramName = String.format("%s:agg", name);
final RexNode probabilityArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue();
final int resolution;
if (aggregateCall.getArgList().size() >= 3) {
final RexNode resolutionArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
resolution = ((Number) RexLiteral.value(resolutionArg)).intValue();
} else {
resolution = ApproximateHistogram.DEFAULT_HISTOGRAM_SIZE;
}
final int numBuckets = ApproximateHistogram.DEFAULT_BUCKET_SIZE;
final float lowerLimit = Float.NEGATIVE_INFINITY;
final float upperLimit = Float.POSITIVE_INFINITY;
// Look for existing matching aggregatorFactory.
for (final Aggregation existing : existingAggregations) {
for (AggregatorFactory factory : existing.getAggregatorFactories()) {
final boolean matches = Aggregations.aggregatorMatches(factory, filter, ApproximateHistogramAggregatorFactory.class, new Predicate<ApproximateHistogramAggregatorFactory>() {
@Override
public boolean apply(final ApproximateHistogramAggregatorFactory theFactory) {
return theFactory.getFieldName().equals(rex.getColumn()) && theFactory.getResolution() == resolution && theFactory.getNumBuckets() == numBuckets && theFactory.getLowerLimit() == lowerLimit && theFactory.getUpperLimit() == upperLimit;
}
});
if (matches) {
// Found existing one. Use this.
return Aggregation.create(ImmutableList.<AggregatorFactory>of(), new QuantilePostAggregator(name, factory.getName(), probability));
}
}
}
if (rowSignature.getColumnType(rex.getColumn()) == ValueType.COMPLEX) {
aggregatorFactory = new ApproximateHistogramFoldingAggregatorFactory(histogramName, rex.getColumn(), resolution, numBuckets, lowerLimit, upperLimit);
} else {
aggregatorFactory = new ApproximateHistogramAggregatorFactory(histogramName, rex.getColumn(), resolution, numBuckets, lowerLimit, upperLimit);
}
return Aggregation.create(ImmutableList.of(aggregatorFactory), new QuantilePostAggregator(name, histogramName, probability)).filter(filter);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveJoinAddNotNullRule method getNotNullConditions.
private static List<RexNode> getNotNullConditions(RelOptCluster cluster, RexBuilder rexBuilder, RelNode input, Set<Integer> inputKeyPositions, Set<String> pushedPredicates) {
final List<RexNode> newConditions = Lists.newArrayList();
for (int pos : inputKeyPositions) {
RelDataType keyType = input.getRowType().getFieldList().get(pos).getType();
// Nothing to do if key cannot be null
if (!keyType.isNullable()) {
continue;
}
RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeInputRef(input, pos));
String digest = cond.toString();
if (pushedPredicates.add(digest)) {
newConditions.add(cond);
}
}
return newConditions;
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveSemiJoinRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
LOG.debug("Matched HiveSemiJoinRule");
final Project project = call.rel(0);
final Join join = call.rel(1);
final RelNode left = call.rel(2);
final Aggregate aggregate = call.rel(3);
final RelOptCluster cluster = join.getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(project.getProjects(), null);
final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount());
if (bits.intersects(rightBits)) {
return;
}
final JoinInfo joinInfo = join.analyzeCondition();
if (!joinInfo.rightSet().equals(ImmutableBitSet.range(aggregate.getGroupCount()))) {
// By the way, neither a super-set nor a sub-set would work.
return;
}
if (join.getJoinType() == JoinRelType.LEFT) {
// since for LEFT join we are only interested in rows from LEFT we can get rid of right side
call.transformTo(call.builder().push(left).project(project.getProjects(), project.getRowType().getFieldNames()).build());
return;
}
if (join.getJoinType() != JoinRelType.INNER) {
return;
}
if (!joinInfo.isEqui()) {
return;
}
LOG.debug("All conditions matched for HiveSemiJoinRule. Going to apply transformation.");
final List<Integer> newRightKeyBuilder = Lists.newArrayList();
final List<Integer> aggregateKeys = aggregate.getGroupSet().asList();
for (int key : joinInfo.rightKeys) {
newRightKeyBuilder.add(aggregateKeys.get(key));
}
final ImmutableIntList newRightKeys = ImmutableIntList.copyOf(newRightKeyBuilder);
final RelNode newRight = aggregate.getInput();
final RexNode newCondition = RelOptUtil.createEquiJoinCondition(left, joinInfo.leftKeys, newRight, newRightKeys, rexBuilder);
RelNode semi = null;
// is not expected further down the pipeline. see jira for more details
if (aggregate.getInput() instanceof HepRelVertex && ((HepRelVertex) aggregate.getInput()).getCurrentRel() instanceof Join) {
Join rightJoin = (Join) (((HepRelVertex) aggregate.getInput()).getCurrentRel());
List<RexNode> projects = new ArrayList<>();
for (int i = 0; i < rightJoin.getRowType().getFieldCount(); i++) {
projects.add(rexBuilder.makeInputRef(rightJoin, i));
}
RelNode topProject = call.builder().push(rightJoin).project(projects, rightJoin.getRowType().getFieldNames(), true).build();
semi = call.builder().push(left).push(topProject).semiJoin(newCondition).build();
} else {
semi = call.builder().push(left).push(aggregate.getInput()).semiJoin(newCondition).build();
}
call.transformTo(call.builder().push(semi).project(project.getProjects(), project.getRowType().getFieldNames()).build());
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode in project hive by apache.
the class HivePointLookupOptimizerRule method onMatch.
public void onMatch(RelOptRuleCall call) {
final Filter filter = call.rel(0);
final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
// 1. We try to transform possible candidates
RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, filter, minNumORClauses);
RexNode newCondition = transformIntoInClause.apply(condition);
// 2. We merge IN expressions
RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
newCondition = mergeInClause.apply(newCondition);
// 3. If we could not transform anything, we bail out
if (newCondition.toString().equals(condition.toString())) {
return;
}
// 4. We create the filter with the new condition
RelNode newFilter = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
call.transformTo(newFilter);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode in project hive by apache.
the class HivePreFilteringRule method extractCommonOperands.
private static List<RexNode> extractCommonOperands(RexBuilder rexBuilder, RexNode condition, int maxCNFNodeCount) {
assert condition.getKind() == SqlKind.OR;
Multimap<String, RexNode> reductionCondition = LinkedHashMultimap.create();
// Data structure to control whether a certain reference is present in every
// operand
Set<String> refsInAllOperands = null;
// 1. We extract the information necessary to create the predicate for the
// new filter; currently we support comparison functions, in and between
ImmutableList<RexNode> operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
for (int i = 0; i < operands.size(); i++) {
final RexNode operand = operands.get(i);
final RexNode operandCNF = RexUtil.toCnf(rexBuilder, maxCNFNodeCount, operand);
final List<RexNode> conjunctions = RelOptUtil.conjunctions(operandCNF);
Set<String> refsInCurrentOperand = Sets.newHashSet();
for (RexNode conjunction : conjunctions) {
// We do not know what it is, we bail out for safety
if (!(conjunction instanceof RexCall) || !HiveCalciteUtil.isDeterministic(conjunction)) {
return new ArrayList<>();
}
RexCall conjCall = (RexCall) conjunction;
RexNode ref = null;
if (COMPARISON.contains(conjCall.getOperator().getKind())) {
if (conjCall.operands.get(0) instanceof RexInputRef && conjCall.operands.get(1) instanceof RexLiteral) {
ref = conjCall.operands.get(0);
} else if (conjCall.operands.get(1) instanceof RexInputRef && conjCall.operands.get(0) instanceof RexLiteral) {
ref = conjCall.operands.get(1);
} else {
// We do not know what it is, we bail out for safety
return new ArrayList<>();
}
} else if (conjCall.getOperator().getKind().equals(SqlKind.IN)) {
ref = conjCall.operands.get(0);
} else if (conjCall.getOperator().getKind().equals(SqlKind.BETWEEN)) {
ref = conjCall.operands.get(1);
} else {
// We do not know what it is, we bail out for safety
return new ArrayList<>();
}
String stringRef = ref.toString();
reductionCondition.put(stringRef, conjCall);
refsInCurrentOperand.add(stringRef);
}
// Updates the references that are present in every operand up till now
if (i == 0) {
refsInAllOperands = refsInCurrentOperand;
} else {
refsInAllOperands = Sets.intersection(refsInAllOperands, refsInCurrentOperand);
}
// bail out
if (refsInAllOperands.isEmpty()) {
return new ArrayList<>();
}
}
// 2. We gather the common factors and return them
List<RexNode> commonOperands = new ArrayList<>();
for (String ref : refsInAllOperands) {
commonOperands.add(RexUtil.composeDisjunction(rexBuilder, reductionCondition.get(ref), false));
}
return commonOperands;
}
Aggregations