use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression in project asterixdb by apache.
the class NestGroupByRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
}
SubplanOperator subplan = (SubplanOperator) op1;
if (subplan.getNestedPlans().size() != 1) {
return false;
}
ILogicalPlan p = subplan.getNestedPlans().get(0);
if (p.getRoots().size() != 1) {
return false;
}
Set<LogicalVariable> free = new HashSet<LogicalVariable>();
OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
if (free.size() != 1) {
return false;
}
LogicalVariable fVar = null;
for (LogicalVariable v : free) {
fVar = v;
break;
}
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
return false;
}
GroupByOperator gby = (GroupByOperator) op2;
if (gby.getNestedPlans().size() != 1) {
return false;
}
ILogicalPlan p2 = gby.getNestedPlans().get(0);
if (p2.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> r2 = p2.getRoots().get(0);
AbstractLogicalOperator opr2 = (AbstractLogicalOperator) r2.getValue();
if (opr2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggOuter = (AggregateOperator) opr2;
int posInAggList = aggOuter.getVariables().indexOf(fVar);
if (posInAggList < 0) {
return false;
}
AbstractLogicalOperator outerAggSon = (AbstractLogicalOperator) aggOuter.getInputs().get(0).getValue();
if (outerAggSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
return false;
}
ILogicalExpression eAgg = aggOuter.getExpressions().get(posInAggList).getValue();
if (eAgg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return false;
}
AbstractFunctionCallExpression listifyCall = (AbstractFunctionCallExpression) eAgg;
if (listifyCall.getFunctionIdentifier() != BuiltinFunctions.LISTIFY) {
return false;
}
ILogicalExpression argListify = listifyCall.getArguments().get(0).getValue();
if (argListify.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return false;
}
Mutable<ILogicalOperator> r = p.getRoots().get(0);
AbstractLogicalOperator opInS = (AbstractLogicalOperator) r.getValue();
if (opInS.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggInner = (AggregateOperator) opInS;
do {
opInS = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
} while (opInS.getOperatorTag() == LogicalOperatorTag.ASSIGN);
if (opInS.getOperatorTag() != LogicalOperatorTag.GROUP) {
return false;
}
AbstractLogicalOperator unnestParent = opInS;
AbstractLogicalOperator opUnder = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
// skip Assigns
while (opUnder.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
unnestParent = opUnder;
opUnder = (AbstractLogicalOperator) opUnder.getInputs().get(0).getValue();
}
if (opUnder.getOperatorTag() != LogicalOperatorTag.UNNEST) {
return false;
}
UnnestOperator unnest = (UnnestOperator) opUnder;
AbstractLogicalOperator unnestSon = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
if (unnestSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
return false;
}
NestedTupleSourceOperator innerNts = (NestedTupleSourceOperator) unnestSon;
ILogicalExpression eUnnest = unnest.getExpressionRef().getValue();
if (eUnnest.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return false;
}
AbstractFunctionCallExpression uf = (AbstractFunctionCallExpression) eUnnest;
if (uf.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
return false;
}
ILogicalExpression scanArg = uf.getArguments().get(0).getValue();
if (scanArg.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return false;
}
if (((VariableReferenceExpression) scanArg).getVariableReference() != fVar) {
return false;
}
LogicalVariable uVar = unnest.getVariable();
GroupByOperator innerGby = (GroupByOperator) opInS;
Set<LogicalVariable> freeInInnerGby = new HashSet<LogicalVariable>();
OperatorPropertiesUtil.getFreeVariablesInSubplans(innerGby, freeInInnerGby);
for (LogicalVariable v : freeInInnerGby) {
if (v != uVar) {
return false;
}
}
unnestParent.getInputs().get(0).setValue(innerNts);
LogicalVariable listifiedVar = ((VariableReferenceExpression) argListify).getVariableReference();
substInSubplan(aggInner, uVar, listifiedVar, context);
gby.getNestedPlans().add(p);
innerNts.getDataSourceReference().setValue(gby);
opRef.setValue(gby);
OperatorPropertiesUtil.typePlan(p, context);
OperatorPropertiesUtil.typePlan(p2, context);
context.computeAndSetTypeEnvironmentForOperator(gby);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression in project asterixdb by apache.
the class AbstractPreclusteredGroupByPOperator method getRequiredPropertiesForChildren.
@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
GroupByOperator gby = (GroupByOperator) op;
StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
if (gby.isGroupAll() && gby.isGlobal()) {
if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
} else {
return emptyUnaryRequirements();
}
}
List<ILocalStructuralProperty> localProps = new ArrayList<>();
Set<LogicalVariable> gbvars = new ListSet<>(columnList);
LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList));
boolean goon = true;
for (ILogicalPlan p : gby.getNestedPlans()) {
// groupings
for (Mutable<ILogicalOperator> r : p.getRoots()) {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
IPhysicalOperator pop2 = op2.getPhysicalOperator();
if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
List<LogicalVariable> gbyColumns = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
List<LogicalVariable> sndOrder = new ArrayList<>();
sndOrder.addAll(gbyColumns);
Set<LogicalVariable> freeVars = new HashSet<>();
try {
OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, freeVars);
} catch (AlgebricksException e) {
throw new IllegalStateException(e);
}
// Only considers group key variables defined out-side the outer-most group-by operator.
sndOrder.retainAll(freeVars);
groupProp.getColumnSet().addAll(sndOrder);
groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
goon = false;
break;
}
}
}
if (!goon) {
break;
}
}
localProps.add(groupProp);
if (reqdByParent != null) {
// propagate parent requirements
List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
if (lpPar != null) {
boolean allOk = true;
List<ILocalStructuralProperty> props = new ArrayList<>(lpPar.size());
for (ILocalStructuralProperty prop : lpPar) {
if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
allOk = false;
break;
}
LocalOrderProperty lop = (LocalOrderProperty) prop;
List<OrderColumn> orderColumns = new ArrayList<>();
List<OrderColumn> ords = lop.getOrderColumns();
for (OrderColumn ord : ords) {
Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
if (p == null) {
p = getDecorPairByRhsVar(gby, ord.getColumn());
if (p == null) {
allOk = false;
break;
}
}
ILogicalExpression e = p.second.getValue();
if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
throw new IllegalStateException("Right hand side of group-by assignment should have been normalized to a variable reference.");
}
LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
orderColumns.add(new OrderColumn(v, ord.getOrder()));
}
props.add(new LocalOrderProperty(orderColumns));
}
List<FunctionalDependency> fdList = new ArrayList<>();
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
List<LogicalVariable> hd = gby.getGbyVarList();
List<LogicalVariable> tl = new ArrayList<>();
tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
fdList.add(new FunctionalDependency(hd, tl));
}
if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, new HashMap<>(), fdList)) {
localProps = props;
}
}
}
IPartitioningProperty pp = null;
AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain());
}
pv[0] = new StructuralPropertiesVector(pp, localProps);
return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}
use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression in project asterixdb by apache.
the class AbstractPreclusteredGroupByPOperator method getLhsGbyVar.
private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) {
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
ILogicalExpression e = ve.second.getValue();
if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
throw new IllegalStateException("Right hand side of group by assignment should have been normalized to a variable reference.");
}
LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
if (v.equals(var)) {
return ve.first;
}
}
return null;
}
use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression in project asterixdb by apache.
the class SubstituteVariableVisitor method visitLimitOperator.
@Override
public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException {
op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
ILogicalExpression offset = op.getOffset().getValue();
if (offset != null) {
offset.substituteVar(pair.first, pair.second);
}
substVarTypes(op, pair);
return null;
}
use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression in project asterixdb by apache.
the class LogicalPropertiesVisitor method visitAssignment.
private void visitAssignment(AbstractAssignOperator op, IOptimizationContext context) throws AlgebricksException {
LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
if (v != null && v.getNumberOfTuples() != null) {
IVariableEvalSizeEnvironment varSizeEnv = context.getVariableEvalSizeEnvironment();
IExpressionEvalSizeComputer evalSize = context.getExpressionEvalSizeComputer();
if (evalSize != null) {
ILogicalOperator op0 = op.getInputs().get(0).getValue();
ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
if (v0 != null) {
long frames0 = v0.getMaxOutputFrames();
// added per tuple
long overhead = 0;
for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
int sz = evalSize.getEvalSize(exprRef.getValue(), varSizeEnv);
if (sz == -1) {
return;
}
overhead += sz;
}
int frameSize = context.getPhysicalOptimizationConfig().getFrameSize();
if (frameSize > 0) {
long sz = frames0 * frameSize + overhead * v.getNumberOfTuples();
int frames1 = (int) (sz / frameSize);
if (sz % frameSize > 0) {
frames1++;
}
v.setMaxOutputFrames(frames1);
}
}
}
}
}
Aggregations