use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class PushSubplanWithAggregateDownThroughProductRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
}
SubplanOperator subplan = (SubplanOperator) op1;
if (subplan.getNestedPlans().size() != 1) {
return false;
}
ILogicalPlan p = subplan.getNestedPlans().get(0);
if (p.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> r = p.getRoots().get(0);
if (((AbstractLogicalOperator) r.getValue()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
Set<LogicalVariable> free = new HashSet<LogicalVariable>();
OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
return false;
}
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
return false;
}
Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
ILogicalOperator b0 = b0Ref.getValue();
List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(b0, b0Scm);
if (b0Scm.containsAll(free)) {
// push subplan on left branch
op2Ref.setValue(b0);
b0Ref.setValue(op1);
opRef.setValue(op2);
return true;
} else {
Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
ILogicalOperator b1 = b1Ref.getValue();
List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(b1, b1Scm);
if (b1Scm.containsAll(free)) {
// push subplan on right branch
op2Ref.setValue(b1);
b1Ref.setValue(op1);
opRef.setValue(op2);
return true;
} else {
return false;
}
}
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class PushAggregateIntoNestedSubplanRule method pushSubplanAsAggIntoNestedSubplan.
private boolean pushSubplanAsAggIntoNestedSubplan(Mutable<ILogicalOperator> subplanOpRef, AbstractOperatorWithNestedPlans nspOp, LogicalVariable varFromNestedAgg, Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex, IOptimizationContext context) throws AlgebricksException {
SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
// only free var can be varFromNestedAgg
HashSet<LogicalVariable> freeVars = new HashSet<>();
OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
for (LogicalVariable vFree : freeVars) {
if (!vFree.equals(varFromNestedAgg)) {
return false;
}
}
List<ILogicalPlan> plans = subplan.getNestedPlans();
if (plans.size() > 1) {
return false;
}
ILogicalPlan p = plans.get(0);
if (p.getRoots().size() > 1) {
return false;
}
Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggInSubplanOp = (AggregateOperator) op;
LogicalVariable unnestVar = null;
boolean pushableNestedSubplan = false;
while (op.getInputs().size() == 1) {
opRef = op.getInputs().get(0);
op = (AbstractLogicalOperator) opRef.getValue();
switch(op.getOperatorTag()) {
case ASSIGN:
break;
case UNNEST:
UnnestOperator unnest = (UnnestOperator) op;
if (unnest.getPositionalVariable() != null) {
// TODO currently subplan with both accumulating and running aggregate is not supported.
return false;
}
ILogicalExpression expr = unnest.getExpressionRef().getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return false;
}
AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
if (fun.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
return false;
}
ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return false;
}
VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
if (!varExpr.getVariableReference().equals(varFromNestedAgg)) {
return false;
}
opRef = op.getInputs().get(0);
op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
return false;
}
pushableNestedSubplan = true;
unnestVar = unnest.getVariable();
break;
default:
return false;
}
}
if (!pushableNestedSubplan) {
return false;
}
for (int i = 0; i < nspOp.getNestedPlans().size(); i++) {
Mutable<ILogicalOperator> nspAggRef = nspOp.getNestedPlans().get(i).getRoots().get(0);
AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue();
Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0);
LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg);
if (listifyVar == null) {
continue;
}
OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context);
nspAgg.getVariables().addAll(aggInSubplanOp.getVariables());
nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
for (LogicalVariable v : aggInSubplanOp.getVariables()) {
nspWithAgg.put(v, nspOp);
nspAggVars.put(v, 0);
nspAggVarToPlanIndex.put(v, i);
}
Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
if (!opRef1InSubplan.getValue().getInputs().isEmpty()) {
Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs();
nspInpList.clear();
nspInpList.add(opRef1InSubplan);
while (true) {
opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
opInpList.clear();
opInpList.add(nspAggChildRef);
break;
}
opRef1InSubplan = opRef2InSubplan;
if (opRef1InSubplan.getValue().getInputs().isEmpty()) {
throw new IllegalStateException("PushAggregateIntoNestedSubplanRule: could not find UNNEST.");
}
}
}
}
subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
OperatorPropertiesUtil.typeOpRec(nspAggRef, context);
}
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class IsomorphismOperatorVisitor method visitSubplanOperator.
@Override
public Boolean visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return Boolean.FALSE;
}
SubplanOperator subplanOpArg = (SubplanOperator) copyAndSubstituteVar(op, arg);
List<ILogicalPlan> plans = op.getNestedPlans();
List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans();
for (int i = 0; i < plans.size(); i++) {
List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
if (roots.size() == rootsArg.size()) {
return Boolean.FALSE;
}
for (int j = 0; j < roots.size(); j++) {
ILogicalOperator topOp1 = roots.get(j).getValue();
ILogicalOperator topOp2 = rootsArg.get(j).getValue();
boolean isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
if (!isomorphic) {
return Boolean.FALSE;
}
}
}
return Boolean.TRUE;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class IntroduceGroupByForSubplanRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
}
SubplanOperator subplan = (SubplanOperator) op0;
Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
ILogicalPlan p = null;
while (plansIter.hasNext()) {
p = plansIter.next();
}
if (p == null) {
return false;
}
if (p.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
Mutable<ILogicalOperator> botRef = subplanRoot;
AbstractLogicalOperator op2;
// Project is optional
if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
op2 = op1;
} else {
ProjectOperator project = (ProjectOperator) op1;
botRef = project.getInputs().get(0);
op2 = (AbstractLogicalOperator) botRef.getValue();
}
if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggregate = (AggregateOperator) op2;
Set<LogicalVariable> free = new HashSet<LogicalVariable>();
VariableUtilities.getUsedVariables(aggregate, free);
Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
while (op3.getInputs().size() == 1) {
Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
VariableUtilities.getProducedVariables(op3, prod);
free.removeAll(prod);
VariableUtilities.getUsedVariables(op3, free);
botRef = op3Ref;
op3Ref = op3.getInputs().get(0);
op3 = (AbstractLogicalOperator) op3Ref.getValue();
}
if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
return false;
}
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
if (join.getCondition().getValue() == ConstantExpression.TRUE) {
return false;
}
VariableUtilities.getUsedVariables(join, free);
AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
// see if there's an NTS at the end of the pipeline
NestedTupleSourceOperator outerNts = getNts(b0);
if (outerNts == null) {
AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
outerNts = getNts(b1);
if (outerNts == null) {
return false;
}
}
Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
if (pkVars == null || pkVars.size() < 1) {
// there is no non-trivial primary key, group-by keys are all live variables
// that were produced by descendant or self
ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
pkVars = new HashSet<LogicalVariable>();
//get live variables
VariableUtilities.getLiveVariables(subplanInput, pkVars);
//get produced variables
Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
//retain the intersection
pkVars.retainAll(producedVars);
}
AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
LogicalVariable testForNull = null;
AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
switch(right.getOperatorTag()) {
case UNNEST:
{
UnnestOperator innerUnnest = (UnnestOperator) right;
// Select [ $y != null ]
testForNull = innerUnnest.getVariable();
break;
}
case RUNNINGAGGREGATE:
{
ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
if (!producedVars.isEmpty()) {
// Select [ $y != null ]
testForNull = producedVars.iterator().next();
}
break;
}
case DATASOURCESCAN:
{
DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
// Select [ $y != null ]
if (innerScan.getVariables().size() == 1) {
testForNull = innerScan.getVariables().get(0);
}
break;
}
default:
break;
}
if (testForNull == null) {
testForNull = context.newVar();
AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
rightRef.setValue(tmpAsgn);
context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
}
IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
GroupByOperator g = new GroupByOperator();
Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
opRef.setValue(g);
selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
prodInpList.clear();
prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
g.getNestedPlans().add(gPlan);
subplanRoot.setValue(op3Ref.getValue());
g.getInputs().add(newSubplanRef);
HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
underVars.removeAll(pkVars);
Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
context.updatePrimaryKeys(mappedVars);
for (LogicalVariable uv : underVars) {
g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
}
OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
context.computeAndSetTypeEnvironmentForOperator(g);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class IntroduceLeftOuterJoinForSubplanRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
}
SubplanOperator subplan = (SubplanOperator) op0;
Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
ILogicalPlan p = null;
while (plansIter.hasNext()) {
p = plansIter.next();
}
if (p == null) {
return false;
}
if (p.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
if (OperatorPropertiesUtil.isMissingTest((AbstractLogicalOperator) opUnder.getValue())) {
return false;
}
switch(op1.getOperatorTag()) {
case INNERJOIN:
{
InnerJoinOperator join = (InnerJoinOperator) op1;
Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
if (ntsRef == null) {
ntsRef = getNtsAtEndOfPipeline(rightRef);
if (ntsRef == null) {
return false;
} else {
Mutable<ILogicalOperator> t = leftRef;
leftRef = rightRef;
rightRef = t;
}
}
ntsRef.setValue(opUnder.getValue());
LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
loj.getInputs().add(leftRef);
loj.getInputs().add(rightRef);
opRef.setValue(loj);
context.computeAndSetTypeEnvironmentForOperator(loj);
return true;
}
case LEFTOUTERJOIN:
{
LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
if (ntsRef == null) {
return false;
}
ntsRef.setValue(opUnder.getValue());
opRef.setValue(join);
context.computeAndSetTypeEnvironmentForOperator(join);
return true;
}
default:
{
return false;
}
}
}
Aggregations