Examples with ListSet org.apache.hyracks.algebricks.common.utils.ListSet used on opensource projects

Example 16 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class AbstractIntroduceGroupByCombinerRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (context.checkIfInDontApplySet(this, op)) {
        return false;
    }
    if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    GroupByOperator gbyOp = (GroupByOperator) op;
    ExecutionMode executionMode = gbyOp.getExecutionMode();
    if (executionMode != ExecutionMode.PARTITIONED && !(executionMode == ExecutionMode.UNPARTITIONED && gbyOp.isGroupAll())) {
        return false;
    }
    BookkeepingInfo bi = new BookkeepingInfo();
    GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
    if (newGbyOp == null) {
        return false;
    }
    Set<LogicalVariable> newGbyLiveVars = new ListSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(newGbyOp, newGbyLiveVars);
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
        List<LogicalVariable> usedDecorVars = new ArrayList<LogicalVariable>();
        // p.second.getValue() should always return a VariableReferenceExpression, hence
        // usedDecorVars should always contain only one variable.
        p.second.getValue().getUsedVariables(usedDecorVars);
        if (!newGbyLiveVars.contains(usedDecorVars.get(0))) {
            // Let the left-hand side of gbyOp's decoration expressions populated through the combiner group-by without
            // any intermediate assignment.
            newGbyOp.addDecorExpression(null, p.second.getValue());
        }
    }
    newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
    Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
    newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
    Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
    newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
    List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
    VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
    Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
    for (LogicalVariable var : freeVars) {
        if (!propagatedVars.contains(var)) {
            LogicalVariable newDecorVar = context.newVar();
            newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
            VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var, newDecorVar, context);
        }
    }
    Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
    opRef3.setValue(newGbyOp);
    typeGby(newGbyOp, context);
    typeGby(gbyOp, context);
    context.addToDontApplySet(this, op);
    return true;
}

Example 17 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class AbstractIntroduceGroupByCombinerRule method tryToPushSubplan.

private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan nestedPlan, GroupByOperator oldGbyOp, GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context) throws AlgebricksException {
    List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
    Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
    for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
        if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, pushedRoots, toReplaceSet)) {
            // For now, if we cannot push everything, give up.
            return new Pair<Boolean, ILogicalPlan>(false, null);
        }
    }
    if (pushedRoots.isEmpty()) {
        return new Pair<Boolean, ILogicalPlan>(true, null);
    } else {
        // Replaces the aggregation expressions in the original group-by op with new ones.
        ILogicalPlan newPlan = new ALogicalPlanImpl(pushedRoots);
        ILogicalPlan plan = fingIdenticalPlan(newGbyOp, newPlan);
        replaceOriginalAggFuncs(toReplaceSet);
        if (plan == null) {
            return new Pair<Boolean, ILogicalPlan>(true, newPlan);
        } else {
            // Does not add a nested subplan to newGbyOp if there already exists an isomorphic plan.
            Set<LogicalVariable> originalVars = new ListSet<LogicalVariable>();
            Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
            for (Mutable<ILogicalOperator> rootRef : pushedRoots) {
                VariableUtilities.getProducedVariables(rootRef.getValue(), originalVars);
            }
            for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
                VariableUtilities.getProducedVariables(rootRef.getValue(), newVars);
            }
            // Replaces variable exprs referring to the variables produced by newPlan by
            // those produced by plan.
            Iterator<LogicalVariable> originalVarIter = originalVars.iterator();
            Iterator<LogicalVariable> newVarIter = newVars.iterator();
            while (originalVarIter.hasNext()) {
                LogicalVariable originalVar = originalVarIter.next();
                LogicalVariable newVar = newVarIter.next();
                for (SimilarAggregatesInfo sai : toReplaceSet) {
                    for (AggregateExprInfo aei : sai.simAggs) {
                        ILogicalExpression afce = aei.aggExprRef.getValue();
                        afce.substituteVar(originalVar, newVar);
                    }
                }
            }
            return new Pair<Boolean, ILogicalPlan>(true, null);
        }
    }
}

Example 18 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class DataSourcePartitioningProvider method computePropertiesVector.

@Override
public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
    IPhysicalPropertiesVector propsVector;
    IPartitioningProperty pp;
    List<ILocalStructuralProperty> propsLocal = new ArrayList<>();
    switch(ds.getDatasourceType()) {
        case DataSource.Type.LOADABLE:
        case DataSource.Type.EXTERNAL_DATASET:
            pp = new RandomPartitioningProperty(domain);
            ds.computeLocalStructuralProperties(propsLocal, scanVariables);
            break;
        case DataSource.Type.FEED:
            pp = getFeedPartitioningProperty(ds, domain, scanVariables);
            break;
        case DataSource.Type.INTERNAL_DATASET:
            Set<LogicalVariable> pvars = new ListSet<>();
            pp = getInternalDatasetPartitioningProperty(ds, domain, scanVariables, pvars);
            propsLocal.add(new LocalOrderProperty(getOrderColumns(pvars)));
            break;
        default:
            throw new IllegalArgumentException();
    }
    propsVector = new StructuralPropertiesVector(pp, propsLocal);
    return propsVector;
}

Example 19 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class ExternalGroupByPOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
    List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
    GroupByOperator gOp = (GroupByOperator) op;
    Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
    if (!columnSet.isEmpty()) {
        propsLocal.add(new LocalGroupingProperty(columnSet));
    }
    for (ILogicalPlan p : gOp.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            ILogicalOperator rOp = r.getValue();
            propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
        }
    }
    ILogicalOperator op2 = op.getInputs().get(0).getValue();
    IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
    deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
}

Example 20 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class FDsAndEquivClassesVisitor method visitGroupByOperator.

@Override
public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException {
    Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
    List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
    ctx.putEquivalenceClassMap(op, equivalenceClasses);
    ctx.putFDList(op, functionalDependencies);
    List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
    for (ILogicalPlan p : op.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            ILogicalOperator op2 = r.getValue();
            equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
            inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
        }
    }
    ILogicalOperator op0 = op.getInputs().get(0).getValue();
    inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
    Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
    for (FunctionalDependency inherited : inheritedFDs) {
        boolean isCoveredByGbyOrDecorVars = true;
        List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead().size());
        for (LogicalVariable v : inherited.getHead()) {
            LogicalVariable vnew = getNewGbyVar(op, v);
            if (vnew == null) {
                vnew = getNewDecorVar(op, v);
                if (vnew == null) {
                    isCoveredByGbyOrDecorVars = false;
                }
                break;
            }
            newHead.add(vnew);
        }
        if (isCoveredByGbyOrDecorVars) {
            List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
            for (LogicalVariable v2 : inherited.getTail()) {
                LogicalVariable v3 = getNewGbyVar(op, v2);
                if (v3 != null) {
                    newTail.add(v3);
                }
            }
            if (!newTail.isEmpty()) {
                FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
                functionalDependencies.add(newFd);
            }
        }
    }
    List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        premiseGby.add(p.first);
    }
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
    LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
        conclDecor.add(GroupByOperator.getDecorVariable(p));
    }
    if (!conclDecor.isEmpty()) {
        functionalDependencies.add(new FunctionalDependency(premiseGby, conclDecor));
    }
    Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        ILogicalExpression expr = p.second.getValue();
        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
            VariableReferenceExpression v = (VariableReferenceExpression) expr;
            gbySet.add(v.getVariableReference());
        }
    }
    LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
    ILocalStructuralProperty normalizedLgp = lgp.normalize(inheritedEcs, inheritedFDs);
    Set<LogicalVariable> normSet = new ListSet<>();
    normalizedLgp.getColumns(normSet);
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<>();
    boolean changed = false;
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        ILogicalExpression expr = p.second.getValue();
        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
            LogicalVariable v2 = varRef.getVariableReference();
            EquivalenceClass ec2 = inheritedEcs.get(v2);
            LogicalVariable v3;
            if (ec2 != null && !ec2.representativeIsConst()) {
                v3 = ec2.getVariableRepresentative();
            } else {
                v3 = v2;
            }
            if (normSet.contains(v3)) {
                newGbyList.add(p);
            } else {
                changed = true;
                decorList.add(p);
            }
        } else {
            newGbyList.add(p);
        }
    }
    if (changed) {
        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Group-by list changed from " + GroupByOperator.veListToString(gByList) + " to " + GroupByOperator.veListToString(newGbyList) + ".\n");
    }
    gByList.clear();
    gByList.addAll(newGbyList);
    return null;
}