Examples with ExchangeOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator used on opensource projects

Example 6 with ExchangeOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testUnPartitionedGroupBy.

@Test
public void testUnPartitionedGroupBy() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a parallel group-by query plan.
    GroupByOperator globalGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    ExchangeOperator exchange = new ExchangeOperator();
    exchange.setPhysicalOperator(new OneToOneExchangePOperator());
    exchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    GroupByOperator localGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    globalGby.getInputs().add(new MutableObject<>(exchange));
    exchange.getInputs().add(new MutableObject<>(localGby));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    globalGby.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 2 * MEMORY_BUDGET + FRAME_SIZE);
}

Example 7 with ExchangeOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method addPartitioningEnforcers.

private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp, IPhysicalPropertiesVector required, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, IOptimizationContext context) throws AlgebricksException {
    if (pp != null) {
        IPhysicalOperator pop;
        switch(pp.getPartitioningType()) {
            case UNPARTITIONED:
                {
                    List<OrderColumn> ordCols = computeOrderColumns(deliveredByChild);
                    if (ordCols.isEmpty()) {
                        pop = new RandomMergeExchangePOperator();
                    } else {
                        if (op.getAnnotations().containsKey(OperatorAnnotations.USE_RANGE_CONNECTOR)) {
                            IRangeMap rangeMap = (IRangeMap) op.getAnnotations().get(OperatorAnnotations.USE_RANGE_CONNECTOR);
                            pop = new RangePartitionMergeExchangePOperator(ordCols, domain, rangeMap);
                        } else {
                            OrderColumn[] sortColumns = new OrderColumn[ordCols.size()];
                            sortColumns = ordCols.toArray(sortColumns);
                            pop = new SortMergeExchangePOperator(sortColumns);
                        }
                    }
                    break;
                }
            case UNORDERED_PARTITIONED:
                {
                    List<LogicalVariable> varList = new ArrayList<>(((UnorderedPartitionedProperty) pp).getColumnSet());
                    String hashMergeHint = context.getMetadataProvider().getConfig().get(HASH_MERGE);
                    if (hashMergeHint == null || !hashMergeHint.equalsIgnoreCase(TRUE_CONSTANT)) {
                        pop = new HashPartitionExchangePOperator(varList, domain);
                        break;
                    }
                    List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
                    List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
                    boolean propWasSet = false;
                    pop = null;
                    if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
                        Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
                        List<FunctionalDependency> fds = context.getFDList(c);
                        if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
                            List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals, cldLocals);
                            pop = new HashPartitionMergeExchangePOperator(orderColumns, varList, domain);
                            propWasSet = true;
                        }
                    }
                    if (!propWasSet) {
                        pop = new HashPartitionExchangePOperator(varList, domain);
                    }
                    break;
                }
            case ORDERED_PARTITIONED:
                {
                    pop = new RangePartitionExchangePOperator(((OrderedPartitionedProperty) pp).getOrderColumns(), domain, null);
                    break;
                }
            case BROADCAST:
                {
                    pop = new BroadcastExchangePOperator(domain);
                    break;
                }
            case RANDOM:
                {
                    RandomPartitioningProperty rpp = (RandomPartitioningProperty) pp;
                    INodeDomain nd = rpp.getNodeDomain();
                    pop = new RandomPartitionExchangePOperator(nd);
                    break;
                }
            default:
                {
                    throw new NotImplementedException("Enforcer for " + pp.getPartitioningType() + " partitioning type has not been implemented.");
                }
        }
        Mutable<ILogicalOperator> ci = op.getInputs().get(i);
        ExchangeOperator exchg = new ExchangeOperator();
        exchg.setPhysicalOperator(pop);
        setNewOp(ci, exchg, context);
        exchg.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(exchg, context);
        context.computeAndSetTypeEnvironmentForOperator(exchg);
        if (AlgebricksConfig.DEBUG) {
            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added partitioning enforcer " + exchg.getPhysicalOperator() + ".\n");
            printOp((AbstractLogicalOperator) op);
        }
    }
}

Example 8 with ExchangeOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator in project asterixdb by apache.

the class ExtractCommonOperatorsRule method rewriteForOneEquivalentClass.

private boolean rewriteForOneEquivalentClass(List<Mutable<ILogicalOperator>> members, IOptimizationContext context) throws AlgebricksException {
    List<Mutable<ILogicalOperator>> group = new ArrayList<Mutable<ILogicalOperator>>();
    boolean rewritten = false;
    while (members.size() > 0) {
        group.clear();
        Mutable<ILogicalOperator> candidate = members.remove(members.size() - 1);
        group.add(candidate);
        for (int i = members.size() - 1; i >= 0; i--) {
            Mutable<ILogicalOperator> peer = members.get(i);
            if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getValue(), peer.getValue())) {
                group.add(peer);
                members.remove(i);
            }
        }
        boolean[] materializationFlags = computeMaterilizationFlags(group);
        if (group.isEmpty()) {
            continue;
        }
        candidate = group.get(0);
        ReplicateOperator rop = new ReplicateOperator(group.size(), materializationFlags);
        rop.setPhysicalOperator(new ReplicatePOperator());
        Mutable<ILogicalOperator> ropRef = new MutableObject<ILogicalOperator>(rop);
        AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue();
        List<Mutable<ILogicalOperator>> originalCandidateParents = childrenToParents.get(candidate);
        rop.setExecutionMode(((AbstractLogicalOperator) candidate.getValue()).getExecutionMode());
        if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
            rop.getInputs().add(candidate);
        } else {
            AbstractLogicalOperator beforeExchange = new ExchangeOperator();
            beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator());
            beforeExchange.setExecutionMode(rop.getExecutionMode());
            Mutable<ILogicalOperator> beforeExchangeRef = new MutableObject<ILogicalOperator>(beforeExchange);
            beforeExchange.getInputs().add(candidate);
            context.computeAndSetTypeEnvironmentForOperator(beforeExchange);
            rop.getInputs().add(beforeExchangeRef);
        }
        context.computeAndSetTypeEnvironmentForOperator(rop);
        for (Mutable<ILogicalOperator> parentRef : originalCandidateParents) {
            AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue();
            int index = parent.getInputs().indexOf(candidate);
            if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
                parent.getInputs().set(index, ropRef);
                rop.getOutputs().add(parentRef);
            } else {
                AbstractLogicalOperator exchange = new ExchangeOperator();
                exchange.setPhysicalOperator(new OneToOneExchangePOperator());
                exchange.setExecutionMode(rop.getExecutionMode());
                MutableObject<ILogicalOperator> exchangeRef = new MutableObject<ILogicalOperator>(exchange);
                exchange.getInputs().add(ropRef);
                rop.getOutputs().add(exchangeRef);
                context.computeAndSetTypeEnvironmentForOperator(exchange);
                parent.getInputs().set(index, exchangeRef);
                context.computeAndSetTypeEnvironmentForOperator(parent);
            }
        }
        List<LogicalVariable> liveVarsNew = new ArrayList<LogicalVariable>();
        VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew);
        ArrayList<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
        for (LogicalVariable liveVar : liveVarsNew) {
            assignExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar)));
        }
        for (Mutable<ILogicalOperator> ref : group) {
            if (ref.equals(candidate)) {
                continue;
            }
            ArrayList<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
            Map<LogicalVariable, LogicalVariable> variableMappingBack = new HashMap<LogicalVariable, LogicalVariable>();
            IsomorphismUtilities.mapVariablesTopDown(ref.getValue(), candidate.getValue(), variableMappingBack);
            for (int i = 0; i < liveVarsNew.size(); i++) {
                liveVars.add(variableMappingBack.get(liveVarsNew.get(i)));
            }
            AbstractLogicalOperator assignOperator = new AssignOperator(liveVars, assignExprs);
            assignOperator.setExecutionMode(rop.getExecutionMode());
            assignOperator.setPhysicalOperator(new AssignPOperator());
            AbstractLogicalOperator projectOperator = new ProjectOperator(liveVars);
            projectOperator.setPhysicalOperator(new StreamProjectPOperator());
            projectOperator.setExecutionMode(rop.getExecutionMode());
            AbstractLogicalOperator exchOp = new ExchangeOperator();
            exchOp.setPhysicalOperator(new OneToOneExchangePOperator());
            exchOp.setExecutionMode(rop.getExecutionMode());
            exchOp.getInputs().add(ropRef);
            MutableObject<ILogicalOperator> exchOpRef = new MutableObject<ILogicalOperator>(exchOp);
            rop.getOutputs().add(exchOpRef);
            assignOperator.getInputs().add(exchOpRef);
            projectOperator.getInputs().add(new MutableObject<ILogicalOperator>(assignOperator));
            // set the types
            context.computeAndSetTypeEnvironmentForOperator(exchOp);
            context.computeAndSetTypeEnvironmentForOperator(assignOperator);
            context.computeAndSetTypeEnvironmentForOperator(projectOperator);
            List<Mutable<ILogicalOperator>> parentOpList = childrenToParents.get(ref);
            for (Mutable<ILogicalOperator> parentOpRef : parentOpList) {
                AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getValue();
                int index = parentOp.getInputs().indexOf(ref);
                ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator : projectOperator;
                if (!HeuristicOptimizer.isHyracksOp(parentOp.getPhysicalOperator().getOperatorTag())) {
                    parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp));
                } else {
                    // If the parent operator is a hyracks operator,
                    // an extra one-to-one exchange is needed.
                    AbstractLogicalOperator exchg = new ExchangeOperator();
                    exchg.setPhysicalOperator(new OneToOneExchangePOperator());
                    exchg.setExecutionMode(childOp.getExecutionMode());
                    exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
                    parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
                    context.computeAndSetTypeEnvironmentForOperator(exchg);
                }
                context.computeAndSetTypeEnvironmentForOperator(parentOp);
            }
        }
        rewritten = true;
    }
    return rewritten;
}

Example 9 with ExchangeOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator in project asterixdb by apache.

the class IsolateHyracksOperatorsRule method insertOneToOneExchange.

private static final void insertOneToOneExchange(Mutable<ILogicalOperator> i, IOptimizationContext context) throws AlgebricksException {
    ExchangeOperator e = new ExchangeOperator();
    e.setPhysicalOperator(new OneToOneExchangePOperator());
    ILogicalOperator inOp = i.getValue();
    e.getInputs().add(new MutableObject<ILogicalOperator>(inOp));
    i.setValue(e);
    // e.recomputeSchema();
    OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(e, context);
    ExecutionMode em = ((AbstractLogicalOperator) inOp).getExecutionMode();
    e.setExecutionMode(em);
    e.computeDeliveredPhysicalProperties(context);
    context.computeAndSetTypeEnvironmentForOperator(e);
}