use of org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveSortMergeRule method onMatch.
// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
final HiveSortLimit topSortLimit = call.rel(0);
final HiveSortLimit bottomSortLimit = call.rel(1);
final RexNode newOffset;
final RexNode newLimit;
if (HiveCalciteUtil.limitRelNode(bottomSortLimit)) {
final RexBuilder rexBuilder = topSortLimit.getCluster().getRexBuilder();
int topOffset = topSortLimit.offset == null ? 0 : RexLiteral.intValue(topSortLimit.offset);
int topLimit = RexLiteral.intValue(topSortLimit.fetch);
int bottomOffset = bottomSortLimit.offset == null ? 0 : RexLiteral.intValue(bottomSortLimit.offset);
int bottomLimit = RexLiteral.intValue(bottomSortLimit.fetch);
// Three different cases
if (topOffset + topLimit <= bottomLimit) {
// 1. Fully contained
// topOffset + topLimit <= bottomLimit
newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
newLimit = topSortLimit.fetch;
} else if (topOffset < bottomLimit) {
// 2. Partially contained
// topOffset + topLimit > bottomLimit && topOffset < bottomLimit
newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomLimit - topOffset));
} else {
// 3. Outside
// we need to create a new limit 0
newOffset = null;
newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0));
}
} else {
// Bottom operator does not contain offset/fetch
newOffset = topSortLimit.offset;
newLimit = topSortLimit.fetch;
}
final HiveSortLimit newSort = bottomSortLimit.copy(bottomSortLimit.getTraitSet(), bottomSortLimit.getInput(), bottomSortLimit.collation, newOffset, newLimit);
call.transformTo(newSort);
}
use of org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveSortUnionReduceRule method onMatch.
public void onMatch(RelOptRuleCall call) {
final HiveSortLimit sort = call.rel(0);
final HiveUnion union = call.rel(1);
List<RelNode> inputs = new ArrayList<>();
// Thus we use 'finishPushSortPastUnion' as a flag to identify if we have finished pushing the
// sort past a union.
boolean finishPushSortPastUnion = true;
final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
for (RelNode input : union.getInputs()) {
// If we do not reduce the input size, we bail out
if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.instance().getRowCount(input)) {
finishPushSortPastUnion = false;
// Here we do some query rewrite. We first get the new fetchRN, which is
// a sum of offset and fetch.
// We then push it through by creating a new branchSort with the new
// fetchRN but no offset.
RexNode fetchRN = sort.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(RexLiteral.intValue(sort.fetch) + offset));
HiveSortLimit branchSort = sort.copy(sort.getTraitSet(), input, sort.getCollation(), null, fetchRN);
branchSort.setRuleCreated(true);
inputs.add(branchSort);
} else {
inputs.add(input);
}
}
// there is nothing to change
if (finishPushSortPastUnion) {
return;
}
// create new union and sort
HiveUnion unionCopy = (HiveUnion) union.copy(union.getTraitSet(), inputs, union.all);
HiveSortLimit result = sort.copy(sort.getTraitSet(), unionCopy, sort.getCollation(), sort.offset, sort.fetch);
call.transformTo(result);
}
use of org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveSubQueryRemoveRule method apply.
protected RexNode apply(RexSubQuery e, Set<CorrelationId> variablesSet, RelOptUtil.Logic logic, HiveSubQRemoveRelBuilder builder, int inputCount, int offset, boolean isCorrScalarAgg) {
switch(e.getKind()) {
case SCALAR_QUERY:
if (isCorrScalarAgg) {
// Transformation :
// Outer Query Left Join (inner query) on correlated predicate and preserve rows only from left side.
builder.push(e.rel);
final List<RexNode> parentQueryFields = new ArrayList<>();
parentQueryFields.addAll(builder.fields());
// id is appended since there could be multiple scalar subqueries and FILTER
// is created using field name
String indicator = "alwaysTrue" + e.rel.getId();
parentQueryFields.add(builder.alias(builder.literal(true), indicator));
builder.project(parentQueryFields);
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
final ImmutableList.Builder<RexNode> operands = ImmutableList.builder();
RexNode literal;
if (isAggZeroOnEmpty(e)) {
literal = builder.literal(0);
} else {
literal = e.rel.getCluster().getRexBuilder().makeNullLiteral(getAggTypeForScalarSub(e));
}
operands.add((builder.isNull(builder.field(indicator))), literal);
operands.add(field(builder, 1, builder.fields().size() - 2));
return builder.call(SqlStdOperatorTable.CASE, operands.build());
}
//Transformation is to left join for correlated predicates and inner join otherwise,
// but do a count on inner side before that to make sure it generates atmost 1 row.
builder.push(e.rel);
// returns single row/column
builder.aggregate(builder.groupKey(), builder.count(false, "cnt"));
SqlFunction countCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT, InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
// we create FILTER (sq_count_check(count()) <= 1) instead of PROJECT because RelFieldTrimmer
// ends up getting rid of Project since it is not used further up the tree
builder.filter(builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, builder.call(countCheck, builder.field("cnt")), builder.literal(1)));
if (!variablesSet.isEmpty()) {
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
} else
builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
builder.push(e.rel);
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
offset++;
return field(builder, inputCount, offset);
case IN:
case EXISTS:
// Most general case, where the left and right keys might have nulls, and
// caller requires 3-valued logic return.
//
// select e.deptno, e.deptno in (select deptno from emp)
//
// becomes
//
// select e.deptno,
// case
// when ct.c = 0 then false
// when dt.i is not null then true
// when e.deptno is null then null
// when ct.ck < ct.c then null
// else false
// end
// from e
// left join (
// (select count(*) as c, count(deptno) as ck from emp) as ct
// cross join (select distinct deptno, true as i from emp)) as dt
// on e.deptno = dt.deptno
//
// If keys are not null we can remove "ct" and simplify to
//
// select e.deptno,
// case
// when dt.i is not null then true
// else false
// end
// from e
// left join (select distinct deptno, true as i from emp) as dt
// on e.deptno = dt.deptno
//
// We could further simplify to
//
// select e.deptno,
// dt.i is not null
// from e
// left join (select distinct deptno, true as i from emp) as dt
// on e.deptno = dt.deptno
//
// but have not yet.
//
// If the logic is TRUE we can just kill the record if the condition
// evaluates to FALSE or UNKNOWN. Thus the query simplifies to an inner
// join:
//
// select e.deptno,
// true
// from e
// inner join (select distinct deptno from emp) as dt
// on e.deptno = dt.deptno
//
builder.push(e.rel);
final List<RexNode> fields = new ArrayList<>();
switch(e.getKind()) {
case IN:
fields.addAll(builder.fields());
// will produce wrong results (because we further rewrite such queries into JOIN)
if (isCorrScalarAgg) {
// returns single row/column
builder.aggregate(builder.groupKey(), builder.count(false, "cnt_in"));
if (!variablesSet.isEmpty()) {
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
} else {
builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
}
SqlFunction inCountCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT, InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
// we create FILTER (sq_count_check(count()) > 0) instead of PROJECT because RelFieldTrimmer
// ends up getting rid of Project since it is not used further up the tree
builder.filter(builder.call(SqlStdOperatorTable.GREATER_THAN, //true here indicates that sq_count_check is for IN/NOT IN subqueries
builder.call(inCountCheck, builder.field("cnt_in"), builder.literal(true)), builder.literal(0)));
offset = offset + 1;
builder.push(e.rel);
}
}
// First, the cross join
switch(logic) {
case TRUE_FALSE_UNKNOWN:
case UNKNOWN_AS_TRUE:
// null keys we do not need to generate count(*), count(c)
if (e.getKind() == SqlKind.EXISTS) {
logic = RelOptUtil.Logic.TRUE_FALSE;
break;
}
builder.aggregate(builder.groupKey(), builder.count(false, "c"), builder.aggregateCall(SqlStdOperatorTable.COUNT, false, null, "ck", builder.fields()));
builder.as("ct");
if (!variablesSet.isEmpty()) {
//builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
} else
builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
offset += 2;
builder.push(e.rel);
break;
}
// Now the left join
switch(logic) {
case TRUE:
if (fields.isEmpty()) {
builder.project(builder.alias(builder.literal(true), "i" + e.rel.getId()));
builder.aggregate(builder.groupKey(0));
} else {
builder.aggregate(builder.groupKey(fields));
}
break;
default:
fields.add(builder.alias(builder.literal(true), "i" + e.rel.getId()));
builder.project(fields);
builder.distinct();
}
builder.as("dt");
final List<RexNode> conditions = new ArrayList<>();
for (Pair<RexNode, RexNode> pair : Pair.zip(e.getOperands(), builder.fields())) {
conditions.add(builder.equals(pair.left, RexUtil.shift(pair.right, offset)));
}
switch(logic) {
case TRUE:
builder.join(JoinRelType.INNER, builder.and(conditions), variablesSet);
return builder.literal(true);
}
builder.join(JoinRelType.LEFT, builder.and(conditions), variablesSet);
final List<RexNode> keyIsNulls = new ArrayList<>();
for (RexNode operand : e.getOperands()) {
if (operand.getType().isNullable()) {
keyIsNulls.add(builder.isNull(operand));
}
}
final ImmutableList.Builder<RexNode> operands = ImmutableList.builder();
switch(logic) {
case TRUE_FALSE_UNKNOWN:
case UNKNOWN_AS_TRUE:
operands.add(builder.equals(builder.field("ct", "c"), builder.literal(0)), builder.literal(false));
//now that we are using LEFT OUTER JOIN to join inner count, count(*)
// with outer table, we wouldn't be able to tell if count is zero
// for inner table since inner join with correlated values will get rid
// of all values where join cond is not true (i.e where actual inner table
// will produce zero result). To handle this case we need to check both
// count is zero or count is null
operands.add((builder.isNull(builder.field("ct", "c"))), builder.literal(false));
break;
}
operands.add(builder.isNotNull(builder.field("dt", "i" + e.rel.getId())), builder.literal(true));
if (!keyIsNulls.isEmpty()) {
//Calcite creates null literal with Null type here but because HIVE doesn't support null type
// it is appropriately typed boolean
operands.add(builder.or(keyIsNulls), e.rel.getCluster().getRexBuilder().makeNullLiteral(SqlTypeName.BOOLEAN));
// we are creating filter here so should not be returning NULL. Not sure why Calcite return NULL
//operands.add(builder.or(keyIsNulls), builder.literal(false));
}
RexNode b = builder.literal(true);
switch(logic) {
case TRUE_FALSE_UNKNOWN:
b = e.rel.getCluster().getRexBuilder().makeNullLiteral(SqlTypeName.BOOLEAN);
// fall through
case UNKNOWN_AS_TRUE:
operands.add(builder.call(SqlStdOperatorTable.LESS_THAN, builder.field("ct", "ck"), builder.field("ct", "c")), b);
break;
}
operands.add(builder.literal(false));
return builder.call(SqlStdOperatorTable.CASE, operands.build());
default:
throw new AssertionError(e.getKind());
}
}
use of org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveUnionPullUpConstantsRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
final Union union = call.rel(0);
final int count = union.getRowType().getFieldCount();
if (count == 1) {
// Project operator.
return;
}
final RexBuilder rexBuilder = union.getCluster().getRexBuilder();
final RelMetadataQuery mq = RelMetadataQuery.instance();
final RelOptPredicateList predicates = mq.getPulledUpPredicates(union);
if (predicates == null) {
return;
}
Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
Map<RexNode, RexNode> constants = new HashMap<>();
for (int i = 0; i < count; i++) {
RexNode expr = rexBuilder.makeInputRef(union, i);
if (conditionsExtracted.containsKey(expr)) {
constants.put(expr, conditionsExtracted.get(expr));
}
}
// None of the expressions are constant. Nothing to do.
if (constants.isEmpty()) {
return;
}
// Create expressions for Project operators before and after the Union
List<RelDataTypeField> fields = union.getRowType().getFieldList();
List<RexNode> topChildExprs = new ArrayList<>();
List<String> topChildExprsFields = new ArrayList<>();
List<RexNode> refs = new ArrayList<>();
ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder();
for (int i = 0; i < count; i++) {
RexNode expr = rexBuilder.makeInputRef(union, i);
RelDataTypeField field = fields.get(i);
if (constants.containsKey(expr)) {
topChildExprs.add(constants.get(expr));
topChildExprsFields.add(field.getName());
} else {
topChildExprs.add(expr);
topChildExprsFields.add(field.getName());
refs.add(expr);
refsIndexBuilder.set(i);
}
}
ImmutableBitSet refsIndex = refsIndexBuilder.build();
// Update top Project positions
final Mappings.TargetMapping mapping = RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse();
topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
// Create new Project-Union-Project sequences
final RelBuilder relBuilder = call.builder();
for (int i = 0; i < union.getInputs().size(); i++) {
RelNode input = union.getInput(i);
List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
for (int j = 0; j < refsIndex.cardinality(); j++) {
int pos = refsIndex.nth(j);
newChildExprs.add(Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos), input.getRowType().getFieldList().get(pos).getName()));
}
if (newChildExprs.isEmpty()) {
// At least a single item in project is required.
newChildExprs.add(Pair.<RexNode, String>of(topChildExprs.get(0), topChildExprsFields.get(0)));
}
// Add the input with project on top
relBuilder.push(input);
relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
}
relBuilder.union(union.all, union.getInputs().size());
// Create top Project fixing nullability of fields
relBuilder.project(topChildExprs, topChildExprsFields);
relBuilder.convert(union.getRowType(), false);
call.transformTo(relBuilder.build());
}
use of org.apache.calcite.rex.RexNode in project hive by apache.
the class HiveAggregateJoinTransposeRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
final Aggregate aggregate = call.rel(0);
final Join join = call.rel(1);
final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
// If any aggregate call has a filter, bail out
for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
if (aggregateCall.getAggregation().unwrap(SqlSplittableAggFunction.class) == null) {
return;
}
if (aggregateCall.filterArg >= 0) {
return;
}
}
// aggregate operator
if (join.getJoinType() != JoinRelType.INNER) {
return;
}
if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) {
return;
}
// Do the columns used by the join appear in the output of the aggregate?
RelMetadataQuery mq = RelMetadataQuery.instance();
final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates);
final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition());
final boolean allColumnsInAggregate = keyColumns.contains(joinColumns);
final ImmutableBitSet belowAggregateColumns = aggregateColumns.union(joinColumns);
// Split join condition
final List<Integer> leftKeys = Lists.newArrayList();
final List<Integer> rightKeys = Lists.newArrayList();
final List<Boolean> filterNulls = Lists.newArrayList();
RexNode nonEquiConj = RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(), join.getCondition(), leftKeys, rightKeys, filterNulls);
// If it contains non-equi join conditions, we bail out
if (!nonEquiConj.isAlwaysTrue()) {
return;
}
// Push each aggregate function down to each side that contains all of its
// arguments. Note that COUNT(*), because it has no arguments, can go to
// both sides.
final Map<Integer, Integer> map = new HashMap<>();
final List<Side> sides = new ArrayList<>();
int uniqueCount = 0;
int offset = 0;
int belowOffset = 0;
for (int s = 0; s < 2; s++) {
final Side side = new Side();
final RelNode joinInput = join.getInput(s);
int fieldCount = joinInput.getRowType().getFieldCount();
final ImmutableBitSet fieldSet = ImmutableBitSet.range(offset, offset + fieldCount);
final ImmutableBitSet belowAggregateKeyNotShifted = belowAggregateColumns.intersect(fieldSet);
for (Ord<Integer> c : Ord.zip(belowAggregateKeyNotShifted)) {
map.put(c.e, belowOffset + c.i);
}
final ImmutableBitSet belowAggregateKey = belowAggregateKeyNotShifted.shift(-offset);
final boolean unique;
if (!allowFunctions) {
assert aggregate.getAggCallList().isEmpty();
// If there are no functions, it doesn't matter as much whether we
// aggregate the inputs before the join, because there will not be
// any functions experiencing a cartesian product effect.
//
// But finding out whether the input is already unique requires a call
// to areColumnsUnique that currently (until [CALCITE-794] "Detect
// cycles when computing statistics" is fixed) places a heavy load on
// the metadata system.
//
// So we choose to imagine the the input is already unique, which is
// untrue but harmless.
//
unique = true;
} else {
final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey);
unique = unique0 != null && unique0;
}
if (unique) {
++uniqueCount;
side.newInput = joinInput;
} else {
List<AggregateCall> belowAggCalls = new ArrayList<>();
final SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry = registry(belowAggCalls);
final Mappings.TargetMapping mapping = s == 0 ? Mappings.createIdentity(fieldCount) : Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
final SqlAggFunction aggregation = aggCall.e.getAggregation();
final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
final AggregateCall call1;
if (fieldSet.contains(ImmutableBitSet.of(aggCall.e.getArgList()))) {
call1 = splitter.split(aggCall.e, mapping);
} else {
call1 = splitter.other(rexBuilder.getTypeFactory(), aggCall.e);
}
if (call1 != null) {
side.split.put(aggCall.i, belowAggregateKey.cardinality() + belowAggCallRegistry.register(call1));
}
}
side.newInput = aggregateFactory.createAggregate(joinInput, false, belowAggregateKey, null, belowAggCalls);
}
offset += fieldCount;
belowOffset += side.newInput.getRowType().getFieldCount();
sides.add(side);
}
if (uniqueCount == 2) {
// invocation of this rule; if we continue we might loop forever.
return;
}
// Update condition
final Mapping mapping = (Mapping) Mappings.target(new Function<Integer, Integer>() {
@Override
public Integer apply(Integer a0) {
return map.get(a0);
}
}, join.getRowType().getFieldCount(), belowOffset);
final RexNode newCondition = RexUtil.apply(mapping, join.getCondition());
// Create new join
RelNode newJoin = joinFactory.createJoin(sides.get(0).newInput, sides.get(1).newInput, newCondition, join.getJoinType(), join.getVariablesStopped(), join.isSemiJoinDone());
// Aggregate above to sum up the sub-totals
final List<AggregateCall> newAggCalls = new ArrayList<>();
final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
final List<RexNode> projects = new ArrayList<>(rexBuilder.identityProjects(newJoin.getRowType()));
for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
final SqlAggFunction aggregation = aggCall.e.getAggregation();
final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
final Integer leftSubTotal = sides.get(0).split.get(aggCall.i);
final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
newAggCalls.add(splitter.topSplit(rexBuilder, registry(projects), groupIndicatorCount, newJoin.getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
}
RelNode r = newJoin;
b: if (allColumnsInAggregate && newAggCalls.isEmpty() && RelOptUtil.areRowTypesEqual(r.getRowType(), aggregate.getRowType(), false)) {
// no need to aggregate
} else {
r = RelOptUtil.createProject(r, projects, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
if (allColumnsInAggregate) {
// let's see if we can convert
List<RexNode> projects2 = new ArrayList<>();
for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
projects2.add(rexBuilder.makeInputRef(r, key));
}
for (AggregateCall newAggCall : newAggCalls) {
final SqlSplittableAggFunction splitter = newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
if (splitter != null) {
projects2.add(splitter.singleton(rexBuilder, r.getRowType(), newAggCall));
}
}
if (projects2.size() == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
// We successfully converted agg calls into projects.
r = RelOptUtil.createProject(r, projects2, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
break b;
}
}
r = aggregateFactory.createAggregate(r, aggregate.indicator, Mappings.apply(mapping, aggregate.getGroupSet()), Mappings.apply2(mapping, aggregate.getGroupSets()), newAggCalls);
}
// Make a cost based decision to pick cheaper plan
RelOptCost afterCost = mq.getCumulativeCost(r);
RelOptCost beforeCost = mq.getCumulativeCost(aggregate);
if (afterCost.isLt(beforeCost)) {
call.transformTo(r);
}
}
Aggregations