Examples with UnorderedPartitionedProperty org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty used on opensource projects

Example 1 with UnorderedPartitionedProperty

use of org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty in project asterixdb by apache.

the class AbstractPreclusteredGroupByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    GroupByOperator gby = (GroupByOperator) op;
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    if (gby.isGroupAll() && gby.isGlobal()) {
        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
        } else {
            return emptyUnaryRequirements();
        }
    }
    List<ILocalStructuralProperty> localProps = new ArrayList<>();
    Set<LogicalVariable> gbvars = new ListSet<>(columnList);
    LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList));
    boolean goon = true;
    for (ILogicalPlan p : gby.getNestedPlans()) {
        // groupings
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
            if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                IPhysicalOperator pop2 = op2.getPhysicalOperator();
                if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
                    List<LogicalVariable> gbyColumns = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
                    List<LogicalVariable> sndOrder = new ArrayList<>();
                    sndOrder.addAll(gbyColumns);
                    Set<LogicalVariable> freeVars = new HashSet<>();
                    try {
                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, freeVars);
                    } catch (AlgebricksException e) {
                        throw new IllegalStateException(e);
                    }
                    // Only considers group key variables defined out-side the outer-most group-by operator.
                    sndOrder.retainAll(freeVars);
                    groupProp.getColumnSet().addAll(sndOrder);
                    groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
                    goon = false;
                    break;
                }
            }
        }
        if (!goon) {
            break;
        }
    }
    localProps.add(groupProp);
    if (reqdByParent != null) {
        // propagate parent requirements
        List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
        if (lpPar != null) {
            boolean allOk = true;
            List<ILocalStructuralProperty> props = new ArrayList<>(lpPar.size());
            for (ILocalStructuralProperty prop : lpPar) {
                if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
                    allOk = false;
                    break;
                }
                LocalOrderProperty lop = (LocalOrderProperty) prop;
                List<OrderColumn> orderColumns = new ArrayList<>();
                List<OrderColumn> ords = lop.getOrderColumns();
                for (OrderColumn ord : ords) {
                    Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
                    if (p == null) {
                        p = getDecorPairByRhsVar(gby, ord.getColumn());
                        if (p == null) {
                            allOk = false;
                            break;
                        }
                    }
                    ILogicalExpression e = p.second.getValue();
                    if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                        throw new IllegalStateException("Right hand side of group-by assignment should have been normalized to a variable reference.");
                    }
                    LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
                    orderColumns.add(new OrderColumn(v, ord.getOrder()));
                }
                props.add(new LocalOrderProperty(orderColumns));
            }
            List<FunctionalDependency> fdList = new ArrayList<>();
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
                List<LogicalVariable> hd = gby.getGbyVarList();
                List<LogicalVariable> tl = new ArrayList<>();
                tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
                fdList.add(new FunctionalDependency(hd, tl));
            }
            if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, new HashMap<>(), fdList)) {
                localProps = props;
            }
        }
    }
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 2 with UnorderedPartitionedProperty

use of org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty in project asterixdb by apache.

the class AbstractHashJoinPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    // In a cost-based optimizer, we would also try to propagate the
    // parent's partitioning requirements.
    IPartitioningProperty pp1;
    IPartitioningProperty pp2;
    switch(partitioningType) {
        case PAIRWISE:
            pp1 = new UnorderedPartitionedProperty(new ListSet<>(keysLeftBranch), context.getComputationNodeDomain());
            pp2 = new UnorderedPartitionedProperty(new ListSet<>(keysRightBranch), context.getComputationNodeDomain());
            break;
        case BROADCAST:
            pp1 = new RandomPartitioningProperty(context.getComputationNodeDomain());
            pp2 = new BroadcastPartitioningProperty(context.getComputationNodeDomain());
            break;
        default:
            throw new IllegalStateException();
    }
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
    pv[0] = OperatorPropertiesUtil.checkUnpartitionedAndGetPropertiesVector(op, new StructuralPropertiesVector(pp1, null));
    pv[1] = OperatorPropertiesUtil.checkUnpartitionedAndGetPropertiesVector(op, new StructuralPropertiesVector(pp2, null));
    IPartitioningRequirementsCoordinator prc;
    switch(kind) {
        case INNER:
            {
                prc = IPartitioningRequirementsCoordinator.EQCLASS_PARTITIONING_COORDINATOR;
                break;
            }
        case LEFT_OUTER:
            {
                prc = new IPartitioningRequirementsCoordinator() {

                    @Override
                    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements, IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context) throws AlgebricksException {
                        if (firstDeliveredPartitioning != null && requirements != null && firstDeliveredPartitioning.getPartitioningType() == requirements.getPartitioningType()) {
                            switch(requirements.getPartitioningType()) {
                                case UNORDERED_PARTITIONED:
                                    {
                                        UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
                                        Set<LogicalVariable> set1 = upp1.getColumnSet();
                                        UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
                                        Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
                                        Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
                                        Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
                                        Set<LogicalVariable> keysCurrent = uppreq.getColumnSet();
                                        List<LogicalVariable> keysFirst = (keysRightBranch.containsAll(keysCurrent)) ? keysRightBranch : keysLeftBranch;
                                        List<LogicalVariable> keysSecond = keysFirst == keysRightBranch ? keysLeftBranch : keysRightBranch;
                                        for (LogicalVariable r : uppreq.getColumnSet()) {
                                            EquivalenceClass ecSnd = eqmap.get(r);
                                            boolean found = false;
                                            int j = 0;
                                            for (LogicalVariable rvar : keysFirst) {
                                                if (rvar == r || ecSnd != null && eqmap.get(rvar) == ecSnd) {
                                                    found = true;
                                                    break;
                                                }
                                                j++;
                                            }
                                            if (!found) {
                                                throw new IllegalStateException("Did not find a variable equivalent to " + r + " among " + keysFirst);
                                            }
                                            LogicalVariable v2 = keysSecond.get(j);
                                            EquivalenceClass ecFst = eqmap.get(v2);
                                            for (LogicalVariable vset1 : set1) {
                                                if (vset1 == v2 || ecFst != null && eqmap.get(vset1) == ecFst) {
                                                    covered.add(vset1);
                                                    modifuppreq.add(r);
                                                    break;
                                                }
                                            }
                                            if (covered.equals(set1)) {
                                                break;
                                            }
                                        }
                                        if (!covered.equals(set1)) {
                                            throw new AlgebricksException("Could not modify " + requirements + " to agree with partitioning property " + firstDeliveredPartitioning + " delivered by previous input operator.");
                                        }
                                        UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq, requirements.getNodeDomain());
                                        return new Pair<>(false, upp2);
                                    }
                                case ORDERED_PARTITIONED:
                                    {
                                        throw new NotImplementedException();
                                    }
                            }
                        }
                        return new Pair<>(true, requirements);
                    }
                };
                break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    return new PhysicalRequirements(pv, prc);
}

Example 3 with UnorderedPartitionedProperty

use of org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty in project asterixdb by apache.

the class RangePartitionMergeExchangePOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
    List<LogicalVariable> varList = new ArrayList<LogicalVariable>();
    for (OrderColumn oc : partitioningFields) {
        varList.add(oc.getColumn());
    }
    IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(varList), domain);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
    List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
    for (ILocalStructuralProperty prop : op2Locals) {
        if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
            locals.add(prop);
        } else {
            break;
        }
    }
    this.deliveredProperties = new StructuralPropertiesVector(p, locals);
}

Example 4 with UnorderedPartitionedProperty

use of org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty in project asterixdb by apache.

the class PreSortedDistinctByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>();
    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
    for (LogicalVariable column : columnList) {
        orderColumns.add(new OrderColumn(column, OrderKind.ASC));
    }
    localProps.add(new LocalOrderProperty(orderColumns));
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 5 with UnorderedPartitionedProperty

use of org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty in project asterixdb by apache.

the class DataSourcePartitioningProvider method getFeedPartitioningProperty.

public static IPartitioningProperty getFeedPartitioningProperty(DataSource ds, INodeDomain domain, List<LogicalVariable> scanVariables) {
    IPartitioningProperty pp;
    if (scanVariables.size() < 2) {
        pp = new RandomPartitioningProperty(domain);
    } else {
        Set<LogicalVariable> pvars = new ListSet<>();
        pvars.addAll(ds.getPrimaryKeyVariables(scanVariables));
        pp = new UnorderedPartitionedProperty(pvars, domain);
    }
    return pp;
}