use of org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder in project asterixdb by apache.
the class EnforceOrderByAfterSubplan method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
if (context.checkIfInDontApplySet(this, op1)) {
return false;
}
List<Mutable<ILogicalOperator>> inputs = op1.getInputs();
context.addToDontApplySet(this, op1);
if (op1.getOperatorTag() == LogicalOperatorTag.ORDER || inputs == null) {
/**
* does not apply if
* 1. there is yet-another order operator on-top-of the subplan, because the downstream order operator's ordering will be broken anyway
* 2. the input operator(s) is null
*/
return false;
}
boolean changed = false;
for (int i = 0; i < inputs.size(); i++) {
Mutable<ILogicalOperator> inputOpRef = inputs.get(i);
AbstractLogicalOperator op = (AbstractLogicalOperator) inputOpRef.getValue();
context.addToDontApplySet(this, op);
if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
continue;
}
/**
* check the order operators whose ordering is not broken before the subplan operator, and then
* duplicate them on-top-of the subplan operator
*/
boolean foundTarget = true;
boolean orderSensitive = false;
Mutable<ILogicalOperator> childRef = op.getInputs().get(0);
AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
while (child.getOperatorTag() != LogicalOperatorTag.ORDER) {
context.addToDontApplySet(this, child);
if (orderBreakingOps.contains(child.getOperatorTag())) {
foundTarget = false;
break;
}
if (child.getOperatorTag() == LogicalOperatorTag.GROUP) {
foundTarget = false;
break;
}
if (orderSensitiveOps.contains(child.getOperatorTag())) {
orderSensitive = true;
}
List<Mutable<ILogicalOperator>> childInputs = child.getInputs();
if (childInputs == null || childInputs.size() > 2 || childInputs.size() < 1) {
foundTarget = false;
break;
} else {
childRef = childInputs.get(0);
child = (AbstractLogicalOperator) childRef.getValue();
}
}
/** the target order-by operator has not been found. */
if (!foundTarget) {
return false;
}
/** copy the original order-by operator and insert on-top-of the subplan operator */
context.addToDontApplySet(this, child);
OrderOperator sourceOrderOp = (OrderOperator) child;
for (Pair<IOrder, Mutable<ILogicalExpression>> expr : sourceOrderOp.getOrderExpressions()) {
if (!expr.second.getValue().isFunctional()) {
return false;
}
}
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = deepCopyOrderAndExpression(sourceOrderOp.getOrderExpressions());
OrderOperator newOrderOp = new OrderOperator(orderExprs);
context.addToDontApplySet(this, newOrderOp);
inputs.set(i, new MutableObject<ILogicalOperator>(newOrderOp));
newOrderOp.getInputs().add(inputOpRef);
context.computeAndSetTypeEnvironmentForOperator(newOrderOp);
if (!orderSensitive) {
/** remove the original order-by */
childRef.setValue(sourceOrderOp.getInputs().get(0).getValue());
}
changed = true;
}
return changed;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder in project asterixdb by apache.
the class InlineLeftNtsInSubplanJoinFlatteningVisitor method visitOrderOperator.
@Override
public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
boolean underJoin = hasJoinAncestor;
visitSingleInputOperator(op);
if (!rewritten || !underJoin) {
return op;
}
// Adjust the ordering if its input operator pipeline has been rewritten.
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprList = new ArrayList<>();
// Adds keyVars to the prefix of sorting columns.
for (LogicalVariable liveVar : liveVarsFromSubplanInput) {
orderExprList.add(new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar))));
}
orderExprList.addAll(op.getOrderExpressions());
// Creates an order operator with the new expression list.
OrderOperator orderOp = new OrderOperator(orderExprList);
orderOp.getInputs().addAll(op.getInputs());
context.computeAndSetTypeEnvironmentForOperator(orderOp);
return orderOp;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder in project asterixdb by apache.
the class PushNestedOrderByUnderPreSortedGroupByRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
return false;
}
if (op.getPhysicalOperator() == null) {
return false;
}
AbstractPhysicalOperator pOp = (AbstractPhysicalOperator) op.getPhysicalOperator();
if (pOp.getOperatorTag() != PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
return false;
}
GroupByOperator gby = (GroupByOperator) op;
ILogicalPlan plan = gby.getNestedPlans().get(0);
AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
if (op2.getOperatorTag() != LogicalOperatorTag.ORDER) {
return false;
}
OrderOperator order1 = (OrderOperator) op2;
if (!isIndependentFromChildren(order1)) {
return false;
}
AbstractPhysicalOperator pOrder1 = (AbstractPhysicalOperator) op2.getPhysicalOperator();
if (pOrder1.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && pOrder1.getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) {
return false;
}
// StableSortPOperator sort1 = (StableSortPOperator) pOrder1;
AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
if (op3.getOperatorTag() != LogicalOperatorTag.ORDER) {
return false;
}
AbstractPhysicalOperator pOp3 = (AbstractPhysicalOperator) op3.getPhysicalOperator();
if (pOp3.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) {
return false;
}
OrderOperator order2 = (OrderOperator) op3;
StableSortPOperator sort2 = (StableSortPOperator) pOp3;
// int k = 0;
for (Pair<IOrder, Mutable<ILogicalExpression>> oe : order1.getOrderExpressions()) {
order2.getOrderExpressions().add(oe);
// sortColumns[n2 + k] = sort1.getSortColumns()[k];
// ++k;
}
// sort2.setSortColumns(sortColumns);
sort2.computeDeliveredProperties(order2, null);
// remove order1
ILogicalOperator underOrder1 = order1.getInputs().get(0).getValue();
opRef2.setValue(underOrder1);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder in project asterixdb by apache.
the class ExtractOrderExpressionsRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.ORDER) {
return false;
}
if (context.checkIfInDontApplySet(this, op1)) {
return false;
}
context.addToDontApplySet(this, op1);
OrderOperator oo = (OrderOperator) op1;
if (!orderHasComplexExpr(oo)) {
return false;
}
Mutable<ILogicalOperator> opRef2 = oo.getInputs().get(0);
for (Pair<IOrder, Mutable<ILogicalExpression>> orderPair : oo.getOrderExpressions()) {
ILogicalExpression expr = orderPair.second.getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE && !AnalysisUtil.isAccessToFieldRecord(expr)) {
LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
orderPair.second.setValue(new VariableReferenceExpression(v));
}
}
context.computeAndSetTypeEnvironmentForOperator(oo);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder in project asterixdb by apache.
the class InlineSubplanInputForNestedTupleSourceRule method applyGeneralFlattening.
private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applyGeneralFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
if (!SubplanFlatteningUtil.containsOperators(subplanOp, ImmutableSet.of(LogicalOperatorTag.DATASOURCESCAN, LogicalOperatorTag.INNERJOIN, // We don't have nested runtime for union-all and distinct hence we have to include them here.
LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.UNIONALL, LogicalOperatorTag.DISTINCT))) {
return new Pair<>(false, new LinkedHashMap<>());
}
Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Creates parameters for the left outer join operator.
Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
ILogicalOperator inputOp = primaryOpAndVars.first;
Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
inputOpRef.setValue(inputOp);
Set<LogicalVariable> inputLiveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs = SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
if (varMap == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, new LinkedHashMap<>());
}
Mutable<ILogicalOperator> lowestAggregateRefInSubplan = SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
ILogicalOperator rightInputOp = rightInputOpRef.getValue();
// Creates a variable to indicate whether a left input tuple is killed in the plan rooted at rightInputOp.
LogicalVariable assignVar = context.newVar();
ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<>(ConstantExpression.TRUE));
assignOp.getInputs().add(rightInputOpRef);
context.computeAndSetTypeEnvironmentForOperator(assignOp);
rightInputOpRef = new MutableObject<>(assignOp);
// Constructs the join predicate for the leftOuter join.
List<Mutable<ILogicalExpression>> joinPredicates = new ArrayList<>();
for (LogicalVariable liveVar : primaryKeyVars) {
List<Mutable<ILogicalExpression>> arguments = new ArrayList<>();
arguments.add(new MutableObject<>(new VariableReferenceExpression(liveVar)));
LogicalVariable rightVar = varMap.get(liveVar);
arguments.add(new MutableObject<>(new VariableReferenceExpression(rightVar)));
ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), arguments);
joinPredicates.add(new MutableObject<>(expr));
}
ILogicalExpression joinExpr = joinPredicates.size() > 1 ? new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), joinPredicates) : joinPredicates.size() > 0 ? joinPredicates.get(0).getValue() : ConstantExpression.TRUE;
LeftOuterJoinOperator leftOuterJoinOp = new LeftOuterJoinOperator(new MutableObject<>(joinExpr), inputOpRef, rightInputOpRef);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(rightInputOp, context);
context.computeAndSetTypeEnvironmentForOperator(leftOuterJoinOp);
// Creates group-by operator.
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
List<ILogicalPlan> nestedPlans = new ArrayList<>();
GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, nestedPlans);
LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
for (LogicalVariable liveVar : primaryKeyVars) {
LogicalVariable newVar = context.newVar();
groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(liveVar))));
// Adds variables for replacements in ancestors.
replacedVarMap.put(liveVar, newVar);
}
for (LogicalVariable liveVar : inputLiveVars) {
if (primaryKeyVars.contains(liveVar)) {
continue;
}
groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
}
// Sets up the nested plan for the groupby operator.
Mutable<ILogicalOperator> aggOpRef = subplanOp.getNestedPlans().get(0).getRoots().get(0);
// Clears the input of the lowest aggregate.
lowestAggregateRefInSubplan.getValue().getInputs().clear();
Mutable<ILogicalOperator> currentOpRef = lowestAggregateRefInSubplan;
// Adds an optional order operator.
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = varMapAndOrderExprs.second;
if (!orderExprs.isEmpty()) {
OrderOperator orderOp = new OrderOperator(orderExprs);
currentOpRef = new MutableObject<>(orderOp);
lowestAggregateRefInSubplan.getValue().getInputs().add(currentOpRef);
}
// Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
// subplan input tuples that are filtered out within a subplan.
Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(assignVar));
List<Mutable<ILogicalExpression>> args = new ArrayList<>();
args.add(filterVarExpr);
List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
SelectOperator selectOp = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)), false, null);
currentOpRef.getValue().getInputs().add(new MutableObject<>(selectOp));
selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<>();
nestedRoots.add(aggOpRef);
nestedPlans.add(new ALogicalPlanImpl(nestedRoots));
groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
// Replaces subplan with the group-by operator.
opRef.setValue(groupbyOp);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
// Recursively applys this rule to the nested plan of the subplan operator,
// for the case where there are nested subplan operators within {@code subplanOp}.
Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(rightInputOpRef, context);
VariableUtilities.substituteVariables(leftOuterJoinOp, result.second, context);
VariableUtilities.substituteVariables(groupbyOp, result.second, context);
// No var mapping from the right input operator should be populated up.
return new Pair<>(true, replacedVarMap);
}
Aggregations