Examples with AbstractLogicalOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator used on opensource projects

Example 41 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class AbstractHashJoinPOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) throws AlgebricksException {
    IPartitioningProperty pp;
    AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
    if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
        AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
        IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
        IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties();
        if (pv0 == null || pv1 == null) {
            pp = null;
        } else {
            pp = pv0.getPartitioningProperty();
        }
    } else {
        pp = IPartitioningProperty.UNPARTITIONED;
    }
    this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context));
}

Example 42 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class EnforceOrderByAfterSubplan method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (context.checkIfInDontApplySet(this, op1)) {
        return false;
    }
    List<Mutable<ILogicalOperator>> inputs = op1.getInputs();
    context.addToDontApplySet(this, op1);
    if (op1.getOperatorTag() == LogicalOperatorTag.ORDER || inputs == null) {
        /**
             * does not apply if
             * 1. there is yet-another order operator on-top-of the subplan, because the downstream order operator's ordering will be broken anyway
             * 2. the input operator(s) is null
             */
        return false;
    }
    boolean changed = false;
    for (int i = 0; i < inputs.size(); i++) {
        Mutable<ILogicalOperator> inputOpRef = inputs.get(i);
        AbstractLogicalOperator op = (AbstractLogicalOperator) inputOpRef.getValue();
        context.addToDontApplySet(this, op);
        if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
            continue;
        }
        /**
             * check the order operators whose ordering is not broken before the subplan operator, and then
             * duplicate them on-top-of the subplan operator
             */
        boolean foundTarget = true;
        boolean orderSensitive = false;
        Mutable<ILogicalOperator> childRef = op.getInputs().get(0);
        AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
        while (child.getOperatorTag() != LogicalOperatorTag.ORDER) {
            context.addToDontApplySet(this, child);
            if (orderBreakingOps.contains(child.getOperatorTag())) {
                foundTarget = false;
                break;
            }
            if (child.getOperatorTag() == LogicalOperatorTag.GROUP) {
                foundTarget = false;
                break;
            }
            if (orderSensitiveOps.contains(child.getOperatorTag())) {
                orderSensitive = true;
            }
            List<Mutable<ILogicalOperator>> childInputs = child.getInputs();
            if (childInputs == null || childInputs.size() > 2 || childInputs.size() < 1) {
                foundTarget = false;
                break;
            } else {
                childRef = childInputs.get(0);
                child = (AbstractLogicalOperator) childRef.getValue();
            }
        }
        /** the target order-by operator has not been found. */
        if (!foundTarget) {
            return false;
        }
        /** copy the original order-by operator and insert on-top-of the subplan operator */
        context.addToDontApplySet(this, child);
        OrderOperator sourceOrderOp = (OrderOperator) child;
        for (Pair<IOrder, Mutable<ILogicalExpression>> expr : sourceOrderOp.getOrderExpressions()) {
            if (!expr.second.getValue().isFunctional()) {
                return false;
            }
        }
        List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = deepCopyOrderAndExpression(sourceOrderOp.getOrderExpressions());
        OrderOperator newOrderOp = new OrderOperator(orderExprs);
        context.addToDontApplySet(this, newOrderOp);
        inputs.set(i, new MutableObject<ILogicalOperator>(newOrderOp));
        newOrderOp.getInputs().add(inputOpRef);
        context.computeAndSetTypeEnvironmentForOperator(newOrderOp);
        if (!orderSensitive) {
            /** remove the original order-by */
            childRef.setValue(sourceOrderOp.getInputs().get(0).getValue());
        }
        changed = true;
    }
    return changed;
}

Example 43 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method physOptimizeOp.

private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required, boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
    boolean changed = false;
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    optimizeUsingConstraintsAndEquivClasses(op);
    PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required, context);
    IPhysicalPropertiesVector[] reqdProperties = null;
    if (pr != null) {
        reqdProperties = pr.getRequiredProperties();
    }
    boolean opIsRedundantSort = false;
    // compute properties and figure out the domain
    INodeDomain childrenDomain = null;
    {
        int j = 0;
        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
            // recursive call
            if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
                changed = true;
            }
            child.computeDeliveredPhysicalProperties(context);
            IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
            if (childrenDomain == null) {
                childrenDomain = delivered.getPartitioningProperty().getNodeDomain();
            } else {
                INodeDomain dom2 = delivered.getPartitioningProperty().getNodeDomain();
                if (!childrenDomain.sameAs(dom2)) {
                    childrenDomain = context.getComputationNodeDomain();
                }
            }
            j++;
        }
    }
    if (reqdProperties != null) {
        for (int k = 0; k < reqdProperties.length; k++) {
            IPhysicalPropertiesVector pv = reqdProperties[k];
            IPartitioningProperty pp = pv.getPartitioningProperty();
            if (pp != null && pp.getNodeDomain() == null) {
                pp.setNodeDomain(childrenDomain);
            }
        }
    }
    // The child index of the child operator to optimize first.
    int startChildIndex = getStartChildIndex(op, pr, nestedPlan, context);
    IPartitioningProperty firstDeliveredPartitioning = null;
    // Enforce data properties in a top-down manner.
    for (int j = 0; j < op.getInputs().size(); j++) {
        // Starts from a partitioning-compatible child if any to loop over all children.
        int childIndex = (j + startChildIndex) % op.getInputs().size();
        IPhysicalPropertiesVector requiredProperty = reqdProperties[childIndex];
        AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
        IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator() + ": " + delivered + "\n");
        IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
        // Coordinates requirements by looking at the firstDeliveredPartitioning.
        Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(requiredProperty.getPartitioningProperty(), firstDeliveredPartitioning, op, context);
        boolean mayExpandPartitioningProperties = pbpp.first;
        IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second, requiredProperty.getLocalProperties());
        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator() + ": " + rqd + "\n");
        // The partitioning property of reqdProperties[childIndex] could be updated here because
        // rqd.getPartitioningProperty() is the same object instance as requiredProperty.getPartitioningProperty().
        IPhysicalPropertiesVector diff = delivered.getUnsatisfiedPropertiesFrom(rqd, mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
        if (isRedundantSort(opRef, delivered, diff, context)) {
            opIsRedundantSort = true;
        }
        if (diff != null) {
            changed = true;
            addEnforcers(op, childIndex, diff, rqd, delivered, childrenDomain, nestedPlan, context);
            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
            if (newChild != child) {
                delivered = newChild.getDeliveredPhysicalProperties();
                IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd, mayExpandPartitioningProperties, context);
                AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
                if (isRedundantSort(opRef, delivered, newDiff, context)) {
                    opIsRedundantSort = true;
                    break;
                }
            }
        }
        if (firstDeliveredPartitioning == null) {
            firstDeliveredPartitioning = delivered.getPartitioningProperty();
        }
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan p : nested.getNestedPlans()) {
            if (physOptimizePlan(p, required, true, context)) {
                changed = true;
            }
        }
    }
    if (opIsRedundantSort) {
        if (AlgebricksConfig.DEBUG) {
            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Removing redundant SORT operator " + op.getPhysicalOperator() + "\n");
            printOp(op);
        }
        changed = true;
        AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
        if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
            nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
        }
        opRef.setValue(nextOp);
        // Now, transfer annotations from the original sort op. to this one.
        AbstractLogicalOperator transferTo = nextOp;
        if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
            // remove duplicate exchange operator
            transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
        }
        transferTo.getAnnotations().putAll(op.getAnnotations());
        physOptimizeOp(opRef, required, nestedPlan, context);
    }
    return changed;
}

Example 44 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method physOptimizePlan.

private boolean physOptimizePlan(ILogicalPlan plan, IPhysicalPropertiesVector pvector, boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
    boolean changed = false;
    for (Mutable<ILogicalOperator> root : plan.getRoots()) {
        if (physOptimizeOp(root, pvector, nestedPlan, context)) {
            changed = true;
        }
        AbstractLogicalOperator op = (AbstractLogicalOperator) root.getValue();
        op.computeDeliveredPhysicalProperties(context);
        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator() + ": " + op.getDeliveredPhysicalProperties() + "\n");
    }
    return changed;
}

Example 45 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class ComplexUnnestToProductRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return false;
    }
    //stop rewriting if the operators originates from a nested tuple source
    if (insideSubplan(opRef)) {
        return false;
    }
    // We may pull selects above the join we create in order to eliminate possible dependencies between
    // the outer and inner input plans of the join.
    List<ILogicalOperator> topSelects = new ArrayList<ILogicalOperator>();
    // Keep track of the operators and used variables participating in the inner input plan.
    HashSet<LogicalVariable> innerUsedVars = new HashSet<LogicalVariable>();
    List<ILogicalOperator> innerOps = new ArrayList<ILogicalOperator>();
    HashSet<LogicalVariable> outerUsedVars = new HashSet<LogicalVariable>();
    List<ILogicalOperator> outerOps = new ArrayList<ILogicalOperator>();
    innerOps.add(op);
    VariableUtilities.getUsedVariables(op, innerUsedVars);
    Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    // Find an unnest or join and partition the plan between the first unnest and that operator into independent parts.
    if (!findPlanPartition(op2, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, false)) {
        // We could not find an unnest or join.
        return false;
    }
    // The last operator must be an unnest or join.
    AbstractLogicalOperator unnestOrJoin = (AbstractLogicalOperator) outerOps.get(outerOps.size() - 1);
    ILogicalOperator outerRoot = null;
    ILogicalOperator innerRoot = null;
    EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
    // If we found a join, simply use it as the outer root.
    if (unnestOrJoin.getOperatorTag() != LogicalOperatorTag.INNERJOIN && unnestOrJoin.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        // We've found a second unnest. First, sanity check that the unnest does not output any live variables
        // that are used by the plan above (until the first unnest).
        List<LogicalVariable> liveVars = new ArrayList<>();
        VariableUtilities.getLiveVariables(unnestOrJoin, liveVars);
        for (LogicalVariable liveVar : liveVars) {
            if (innerUsedVars.contains(liveVar)) {
                return false;
            }
        }
        // Continue finding a partitioning of the plan such that the inner and outer partitions are independent, in order to feed a join.
        // Now, we look below the second unnest or join.
        VariableUtilities.getUsedVariables(unnestOrJoin, outerUsedVars);
        AbstractLogicalOperator unnestChild = (AbstractLogicalOperator) unnestOrJoin.getInputs().get(0).getValue();
        if (!findPlanPartition(unnestChild, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, true)) {
            // We could not find a suitable partitioning.
            return false;
        }
    }
    innerRoot = buildOperatorChain(innerOps, ets, context);
    context.computeAndSetTypeEnvironmentForOperator(innerRoot);
    outerRoot = buildOperatorChain(outerOps, null, context);
    context.computeAndSetTypeEnvironmentForOperator(outerRoot);
    InnerJoinOperator product = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
    // Outer branch.
    product.getInputs().add(new MutableObject<ILogicalOperator>(outerRoot));
    // Inner branch.
    product.getInputs().add(new MutableObject<ILogicalOperator>(innerRoot));
    context.computeAndSetTypeEnvironmentForOperator(product);
    // Put the selects on top of the join.
    ILogicalOperator topOp = product;
    if (!topSelects.isEmpty()) {
        topOp = buildOperatorChain(topSelects, product, context);
    }
    // Plug the selects + product in the plan.
    opRef.setValue(topOp);
    context.computeAndSetTypeEnvironmentForOperator(topOp);
    return true;
}