Examples with InnerJoinOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator used on opensource projects

Example 6 with InnerJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator in project asterixdb by apache.

the class PushSelectIntoJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>();
    Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>();
    Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>();
    Collection<LogicalVariable> liveInOpsToPushRight = new HashSet<LogicalVariable>();
    List<ILogicalOperator> pushedOnLeft = new ArrayList<ILogicalOperator>();
    List<ILogicalOperator> pushedOnRight = new ArrayList<ILogicalOperator>();
    List<ILogicalOperator> pushedOnEither = new ArrayList<ILogicalOperator>();
    LinkedList<ILogicalOperator> notPushedStack = new LinkedList<ILogicalOperator>();
    Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
    Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
        return false;
    }
    SelectOperator select = (SelectOperator) op;
    Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
    AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue();
    AbstractLogicalOperator op2 = son;
    boolean needToPushOps = false;
    while (son.isMap()) {
        needToPushOps = true;
        Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0);
        son = (AbstractLogicalOperator) opRefLink.getValue();
    }
    if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN && son.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN;
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son;
    Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0);
    Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1);
    if (needToPushOps) {
        ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue();
        ILogicalOperator joinBranchRight = joinBranchRightRef.getValue();
        VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft);
        VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight);
        Mutable<ILogicalOperator> opIterRef = opRef2;
        ILogicalOperator opIter = op2;
        while (opIter != join) {
            LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag();
            if (tag == LogicalOperatorTag.PROJECT) {
                notPushedStack.addFirst(opIter);
            } else {
                VariableUtilities.getUsedVariables(opIter, usedVars);
                VariableUtilities.getProducedVariables(opIter, producedVars);
                if (usedVars.size() == 0) {
                    pushedOnEither.add(opIter);
                } else if (joinLiveVarsLeft.containsAll(usedVars)) {
                    pushedOnLeft.add(opIter);
                    liveInOpsToPushLeft.addAll(producedVars);
                } else if (joinLiveVarsRight.containsAll(usedVars)) {
                    pushedOnRight.add(opIter);
                    liveInOpsToPushRight.addAll(producedVars);
                } else {
                    return false;
                }
            }
            opIterRef = opIter.getInputs().get(0);
            opIter = opIterRef.getValue();
        }
        if (isLoj && pushedOnLeft.isEmpty()) {
            return false;
        }
    }
    boolean intersectsAllBranches = true;
    boolean[] intersectsBranch = new boolean[join.getInputs().size()];
    LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>();
    select.getCondition().getValue().getUsedVariables(selectVars);
    int i = 0;
    for (Mutable<ILogicalOperator> branch : join.getInputs()) {
        LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>();
        VariableUtilities.getLiveVariables(branch.getValue(), branchVars);
        if (i == 0) {
            branchVars.addAll(liveInOpsToPushLeft);
        } else {
            branchVars.addAll(liveInOpsToPushRight);
        }
        if (OperatorPropertiesUtil.disjoint(selectVars, branchVars)) {
            intersectsAllBranches = false;
        } else {
            intersectsBranch[i] = true;
        }
        i++;
    }
    if (!intersectsBranch[0] && !intersectsBranch[1]) {
        return false;
    }
    if (needToPushOps) {
        //We should push independent ops into the first branch that the selection depends on
        if (intersectsBranch[0]) {
            pushOps(pushedOnEither, joinBranchLeftRef, context);
        } else {
            pushOps(pushedOnEither, joinBranchRightRef, context);
        }
        pushOps(pushedOnLeft, joinBranchLeftRef, context);
        pushOps(pushedOnRight, joinBranchRightRef, context);
    }
    if (intersectsAllBranches) {
        addCondToJoin(select, join, context);
    } else {
        // push down
        Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
        ILogicalExpression selectCondition = select.getCondition().getValue();
        boolean lojToInner = false;
        for (int j = 0; j < intersectsBranch.length; j++) {
            Mutable<ILogicalOperator> branch = branchIter.next();
            boolean inter = intersectsBranch[j];
            if (!inter) {
                continue;
            }
            // to inner join for this case.
            if (j > 0 && isLoj && containsNotMissingFiltering(selectCondition)) {
                lojToInner = true;
            }
            if ((j > 0 && isLoj) && containsMissingFiltering(selectCondition)) {
                // Select "is-not-missing($$var)" cannot be pushed in the right branch of a LOJ;
                notPushedStack.addFirst(select);
            } else {
                // Conditions for the left branch can always be pushed.
                // Other conditions can be pushed to the right branch of a LOJ.
                copySelectToBranch(select, branch, context);
            }
        }
        if (lojToInner) {
            // Rewrites left outer join  to inner join.
            InnerJoinOperator innerJoin = new InnerJoinOperator(join.getCondition());
            innerJoin.getInputs().addAll(join.getInputs());
            join = innerJoin;
            context.computeAndSetTypeEnvironmentForOperator(join);
        }
    }
    ILogicalOperator top = join;
    for (ILogicalOperator npOp : notPushedStack) {
        List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs();
        npInpList.clear();
        npInpList.add(new MutableObject<ILogicalOperator>(top));
        context.computeAndSetTypeEnvironmentForOperator(npOp);
        top = npOp;
    }
    opRef.setValue(top);
    return true;
}

Example 7 with InnerJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator in project asterixdb by apache.

the class IntroduceLeftOuterJoinForSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op0;
    Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
    Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
    if (OperatorPropertiesUtil.isMissingTest((AbstractLogicalOperator) opUnder.getValue())) {
        return false;
    }
    switch(op1.getOperatorTag()) {
        case INNERJOIN:
            {
                InnerJoinOperator join = (InnerJoinOperator) op1;
                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
                Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
                if (ntsRef == null) {
                    ntsRef = getNtsAtEndOfPipeline(rightRef);
                    if (ntsRef == null) {
                        return false;
                    } else {
                        Mutable<ILogicalOperator> t = leftRef;
                        leftRef = rightRef;
                        rightRef = t;
                    }
                }
                ntsRef.setValue(opUnder.getValue());
                LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
                loj.getInputs().add(leftRef);
                loj.getInputs().add(rightRef);
                opRef.setValue(loj);
                context.computeAndSetTypeEnvironmentForOperator(loj);
                return true;
            }
        case LEFTOUTERJOIN:
            {
                LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
                if (ntsRef == null) {
                    return false;
                }
                ntsRef.setValue(opUnder.getValue());
                opRef.setValue(join);
                context.computeAndSetTypeEnvironmentForOperator(join);
                return true;
            }
        default:
            {
                return false;
            }
    }
}

Example 8 with InnerJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator in project asterixdb by apache.

the class NestedSubplanToJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    if (context.checkIfInDontApplySet(this, opRef.getValue()))
        return false;
    context.addToDontApplySet(this, opRef.getValue());
    ILogicalOperator op1 = opRef.getValue();
    if (op1.getInputs().size() == 0) {
        return false;
    }
    boolean rewritten = false;
    for (int index = 0; index < op1.getInputs().size(); index++) {
        AbstractLogicalOperator child = (AbstractLogicalOperator) op1.getInputs().get(index).getValue();
        if (child.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
            continue;
        }
        AbstractOperatorWithNestedPlans subplan = (AbstractOperatorWithNestedPlans) child;
        Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
        OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
        if (!freeVars.isEmpty()) {
            /**
                 * the subplan is correlated with the outer plan, other rules can deal with it
                 */
            continue;
        }
        /** get the input operator of the subplan operator */
        ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
        AbstractLogicalOperator subplanInputOp = (AbstractLogicalOperator) subplanInput;
        /** If the other join branch is a trivial plan, do not do the rewriting. */
        if (subplanInputOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
            continue;
        }
        /** get all nested top operators */
        List<ILogicalPlan> nestedPlans = subplan.getNestedPlans();
        List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<Mutable<ILogicalOperator>>();
        for (ILogicalPlan nestedPlan : nestedPlans) {
            nestedRoots.addAll(nestedPlan.getRoots());
        }
        if (nestedRoots.size() == 0) {
            /** there is no nested top operators */
            continue;
        }
        /**
             * Expends the input and roots into a DAG of nested loop joins.
             * Though joins should be left-outer joins, a left-outer join with condition TRUE is equivalent to an inner join.
             **/
        Mutable<ILogicalExpression> expr = new MutableObject<ILogicalExpression>(ConstantExpression.TRUE);
        Mutable<ILogicalOperator> nestedRootRef = nestedRoots.get(0);
        ILogicalOperator join = new InnerJoinOperator(expr, new MutableObject<ILogicalOperator>(subplanInput), nestedRootRef);
        /** rewrite the nested tuple source to be empty tuple source */
        rewriteNestedTupleSource(nestedRootRef, context);
        for (int i = 1; i < nestedRoots.size(); i++) {
            join = new InnerJoinOperator(expr, new MutableObject<ILogicalOperator>(join), nestedRoots.get(i));
        }
        op1.getInputs().get(index).setValue(join);
        context.computeAndSetTypeEnvironmentForOperator(join);
        rewritten = true;
    }
    return rewritten;
}

Example 9 with InnerJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator in project asterixdb by apache.

the class DisjunctivePredicateToJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
    if (metadataProvider.isBlockingOperatorDisabled()) {
        return false;
    }
    SelectOperator select;
    if ((select = asSelectOperator(opRef)) == null) {
        return false;
    }
    AbstractFunctionCallExpression condEx;
    if ((condEx = asFunctionCallExpression(select.getCondition(), AlgebricksBuiltinFunctions.OR)) == null) {
        return false;
    }
    List<Mutable<ILogicalExpression>> args = condEx.getArguments();
    VariableReferenceExpression varEx = null;
    IAType valType = null;
    HashSet<AsterixConstantValue> values = new HashSet<AsterixConstantValue>();
    for (Mutable<ILogicalExpression> arg : args) {
        AbstractFunctionCallExpression fctCall;
        if ((fctCall = asFunctionCallExpression(arg, AlgebricksBuiltinFunctions.EQ)) == null) {
            return false;
        }
        boolean haveConst = false;
        boolean haveVar = false;
        List<Mutable<ILogicalExpression>> fctArgs = fctCall.getArguments();
        for (Mutable<ILogicalExpression> fctArg : fctArgs) {
            final ILogicalExpression argExpr = fctArg.getValue();
            switch(argExpr.getExpressionTag()) {
                case CONSTANT:
                    haveConst = true;
                    AsterixConstantValue value = (AsterixConstantValue) ((ConstantExpression) argExpr).getValue();
                    if (valType == null) {
                        valType = value.getObject().getType();
                    } else if (!isCompatible(valType, value.getObject().getType())) {
                        return false;
                    }
                    values.add(value);
                    break;
                case VARIABLE:
                    haveVar = true;
                    final VariableReferenceExpression varArg = (VariableReferenceExpression) argExpr;
                    if (varEx == null) {
                        varEx = varArg;
                    } else if (!varEx.getVariableReference().equals(varArg.getVariableReference())) {
                        return false;
                    }
                    break;
                default:
                    return false;
            }
        }
        if (!(haveVar && haveConst)) {
            return false;
        }
    }
    AOrderedList list = new AOrderedList(new AOrderedListType(valType, "orderedlist"));
    for (AsterixConstantValue value : values) {
        list.add(value.getObject());
    }
    EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
    context.computeAndSetTypeEnvironmentForOperator(ets);
    ILogicalExpression cExp = new ConstantExpression(new AsterixConstantValue(list));
    Mutable<ILogicalExpression> mutCExp = new MutableObject<>(cExp);
    IFunctionInfo scanFctInfo = BuiltinFunctions.getAsterixFunctionInfo(BuiltinFunctions.SCAN_COLLECTION);
    UnnestingFunctionCallExpression scanExp = new UnnestingFunctionCallExpression(scanFctInfo, mutCExp);
    LogicalVariable scanVar = context.newVar();
    UnnestOperator unn = new UnnestOperator(scanVar, new MutableObject<>(scanExp));
    unn.getInputs().add(new MutableObject<>(ets));
    context.computeAndSetTypeEnvironmentForOperator(unn);
    IFunctionInfo eqFctInfo = BuiltinFunctions.getAsterixFunctionInfo(AlgebricksBuiltinFunctions.EQ);
    AbstractFunctionCallExpression eqExp = new ScalarFunctionCallExpression(eqFctInfo);
    eqExp.getArguments().add(new MutableObject<>(new VariableReferenceExpression(scanVar)));
    eqExp.getArguments().add(new MutableObject<>(varEx.cloneExpression()));
    eqExp.getAnnotations().put(IndexedNLJoinExpressionAnnotation.INSTANCE, IndexedNLJoinExpressionAnnotation.INSTANCE);
    BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
    // Broadcast the OR predicates branch.
    bcast.setObject(BroadcastExpressionAnnotation.BroadcastSide.LEFT);
    eqExp.getAnnotations().put(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast);
    InnerJoinOperator jOp = new InnerJoinOperator(new MutableObject<>(eqExp));
    jOp.getInputs().add(new MutableObject<>(unn));
    jOp.getInputs().add(select.getInputs().get(0));
    opRef.setValue(jOp);
    context.computeAndSetTypeEnvironmentForOperator(jOp);
    return true;
}

Example 10 with InnerJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator in project asterixdb by apache.

the class PushGroupByThroughProduct method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
        return false;
    }
    InnerJoinOperator join = (InnerJoinOperator) op2;
    if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
        // not a product
        return false;
    }
    GroupByOperator gby = (GroupByOperator) op1;
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    Mutable<ILogicalOperator> opLeftRef = join.getInputs().get(0);
    ILogicalOperator opLeft = opLeftRef.getValue();
    switch(canPushThrough(gby, opLeft, decorToPush, decorNotToPush)) {
        case REPEATED_DECORS:
            {
                return false;
            }
        case TRUE:
            {
                push(opRef, opRef2, 0, decorToPush, decorNotToPush, context);
                return true;
            }
        case FALSE:
            {
                decorToPush.clear();
                Mutable<ILogicalOperator> opRightRef = join.getInputs().get(1);
                ILogicalOperator opRight = opRightRef.getValue();
                if (canPushThrough(gby, opRight, decorToPush, decorNotToPush) == PushTestResult.TRUE) {
                    push(opRef, opRef2, 1, decorToPush, decorNotToPush, context);
                    return true;
                } else {
                    return false;
                }
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
}