Examples with GroupByOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator used on opensource projects

Example 6 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testParallelJoin.

@Test
public void testParallelJoin() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a join query plan.
    InnerJoinOperator join = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
    // Left child plan of the join.
    ExchangeOperator leftChildExchange = new ExchangeOperator();
    leftChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
    leftChildExchange.setPhysicalOperator(new HashPartitionExchangePOperator(Collections.emptyList(), null));
    InnerJoinOperator leftChild = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
    join.getInputs().add(new MutableObject<>(leftChildExchange));
    leftChildExchange.getInputs().add(new MutableObject<>(leftChild));
    EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
    ets.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
    leftChild.getInputs().add(new MutableObject<>(ets));
    leftChild.getInputs().add(new MutableObject<>(ets));
    // Right child plan of the join.
    ExchangeOperator rightChildExchange = new ExchangeOperator();
    rightChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
    rightChildExchange.setPhysicalOperator(new HashPartitionExchangePOperator(Collections.emptyList(), null));
    GroupByOperator rightChild = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.LOCAL);
    join.getInputs().add(new MutableObject<>(rightChildExchange));
    rightChildExchange.getInputs().add(new MutableObject<>(rightChild));
    rightChild.getInputs().add(new MutableObject<>(ets));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    join.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == PARALLELISM);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 3 * MEMORY_BUDGET * PARALLELISM + 2 * 2L * PARALLELISM * PARALLELISM * FRAME_SIZE + 3 * FRAME_SIZE * PARALLELISM);
}

Example 7 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RequiredCapacityVisitorTest method makeGroupByOperator.

private GroupByOperator makeGroupByOperator(AbstractLogicalOperator.ExecutionMode exeMode) {
    GroupByOperator groupByOperator = new GroupByOperator();
    groupByOperator.setExecutionMode(exeMode);
    return groupByOperator;
}

Example 8 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testUnPartitionedJoin.

@Test
public void testUnPartitionedJoin() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a join query plan.
    InnerJoinOperator join = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    // Left child plan of the join.
    ExchangeOperator leftChildExchange = new ExchangeOperator();
    leftChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    leftChildExchange.setPhysicalOperator(new OneToOneExchangePOperator());
    InnerJoinOperator leftChild = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    join.getInputs().add(new MutableObject<>(leftChildExchange));
    leftChildExchange.getInputs().add(new MutableObject<>(leftChild));
    EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
    ets.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    leftChild.getInputs().add(new MutableObject<>(ets));
    leftChild.getInputs().add(new MutableObject<>(ets));
    // Right child plan of the join.
    ExchangeOperator rightChildExchange = new ExchangeOperator();
    rightChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    rightChildExchange.setPhysicalOperator(new OneToOneExchangePOperator());
    GroupByOperator rightChild = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    join.getInputs().add(new MutableObject<>(rightChildExchange));
    rightChildExchange.getInputs().add(new MutableObject<>(rightChild));
    rightChild.getInputs().add(new MutableObject<>(ets));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    join.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 3 * MEMORY_BUDGET + 5L * FRAME_SIZE);
}

Example 9 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class AbstractPreclusteredGroupByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    GroupByOperator gby = (GroupByOperator) op;
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    if (gby.isGroupAll() && gby.isGlobal()) {
        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
        } else {
            return emptyUnaryRequirements();
        }
    }
    List<ILocalStructuralProperty> localProps = new ArrayList<>();
    Set<LogicalVariable> gbvars = new ListSet<>(columnList);
    LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList));
    boolean goon = true;
    for (ILogicalPlan p : gby.getNestedPlans()) {
        // groupings
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
            if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                IPhysicalOperator pop2 = op2.getPhysicalOperator();
                if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
                    List<LogicalVariable> gbyColumns = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
                    List<LogicalVariable> sndOrder = new ArrayList<>();
                    sndOrder.addAll(gbyColumns);
                    Set<LogicalVariable> freeVars = new HashSet<>();
                    try {
                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, freeVars);
                    } catch (AlgebricksException e) {
                        throw new IllegalStateException(e);
                    }
                    // Only considers group key variables defined out-side the outer-most group-by operator.
                    sndOrder.retainAll(freeVars);
                    groupProp.getColumnSet().addAll(sndOrder);
                    groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
                    goon = false;
                    break;
                }
            }
        }
        if (!goon) {
            break;
        }
    }
    localProps.add(groupProp);
    if (reqdByParent != null) {
        // propagate parent requirements
        List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
        if (lpPar != null) {
            boolean allOk = true;
            List<ILocalStructuralProperty> props = new ArrayList<>(lpPar.size());
            for (ILocalStructuralProperty prop : lpPar) {
                if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
                    allOk = false;
                    break;
                }
                LocalOrderProperty lop = (LocalOrderProperty) prop;
                List<OrderColumn> orderColumns = new ArrayList<>();
                List<OrderColumn> ords = lop.getOrderColumns();
                for (OrderColumn ord : ords) {
                    Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
                    if (p == null) {
                        p = getDecorPairByRhsVar(gby, ord.getColumn());
                        if (p == null) {
                            allOk = false;
                            break;
                        }
                    }
                    ILogicalExpression e = p.second.getValue();
                    if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                        throw new IllegalStateException("Right hand side of group-by assignment should have been normalized to a variable reference.");
                    }
                    LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
                    orderColumns.add(new OrderColumn(v, ord.getOrder()));
                }
                props.add(new LocalOrderProperty(orderColumns));
            }
            List<FunctionalDependency> fdList = new ArrayList<>();
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
                List<LogicalVariable> hd = gby.getGbyVarList();
                List<LogicalVariable> tl = new ArrayList<>();
                tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
                fdList.add(new FunctionalDependency(hd, tl));
            }
            if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, new HashMap<>(), fdList)) {
                localProps = props;
            }
        }
    }
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 10 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class IsomorphismVariableMappingVisitor method mapVariablesForGroupBy.

private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
    if (left.getOperatorTag() != right.getOperatorTag()) {
        return;
    }
    GroupByOperator leftOp = (GroupByOperator) left;
    GroupByOperator rightOp = (GroupByOperator) right;
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList();
    mapVarExprPairList(leftPairs, rightPairs);
    leftPairs = leftOp.getDecorList();
    rightPairs = rightOp.getDecorList();
    mapVarExprPairList(leftPairs, rightPairs);
}