Examples with LeftOuterJoinOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator used on opensource projects

Example 6 with LeftOuterJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator in project asterixdb by apache.

the class InlineLeftNtsInSubplanJoinFlatteningVisitor method visitInnerJoinOperator.

@Override
public ILogicalOperator visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
    hasJoinAncestor = true;
    boolean needToSwitch = false;
    for (int i = 0; i < op.getInputs().size(); ++i) {
        // Deals with single input operators.
        ILogicalOperator newChild = op.getInputs().get(i).getValue().accept(this, null);
        op.getInputs().get(i).setValue(newChild);
        if (i == 1) {
            needToSwitch = true;
        }
        if (rewritten) {
            break;
        }
    }
    // Checks whether there is a need to switch two join branches.
    if (rewritten && needToSwitch) {
        Mutable<ILogicalOperator> leftBranch = op.getInputs().get(0);
        Mutable<ILogicalOperator> rightBranch = op.getInputs().get(1);
        op.getInputs().set(0, rightBranch);
        op.getInputs().set(1, leftBranch);
    }
    AbstractBinaryJoinOperator returnOp = op;
    // After rewriting, the original inner join should become an left outer join.
    if (rewritten) {
        returnOp = new LeftOuterJoinOperator(op.getCondition());
        returnOp.getInputs().addAll(op.getInputs());
        injectNullCheckVars(returnOp);
    }
    return returnOp;
}

Example 7 with LeftOuterJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator in project asterixdb by apache.

the class InlineSubplanInputForNestedTupleSourceRule method applyGeneralFlattening.

private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applyGeneralFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
    if (!SubplanFlatteningUtil.containsOperators(subplanOp, ImmutableSet.of(LogicalOperatorTag.DATASOURCESCAN, LogicalOperatorTag.INNERJOIN, // We don't have nested runtime for union-all and distinct hence we have to include them here.
    LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.UNIONALL, LogicalOperatorTag.DISTINCT))) {
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
    ILogicalOperator inputOpBackup = inputOpRef.getValue();
    // Creates parameters for the left outer join operator.
    Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
    ILogicalOperator inputOp = primaryOpAndVars.first;
    Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
    inputOpRef.setValue(inputOp);
    Set<LogicalVariable> inputLiveVars = new HashSet<>();
    VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
    Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs = SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
    Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
    if (varMap == null) {
        inputOpRef.setValue(inputOpBackup);
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> lowestAggregateRefInSubplan = SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
    Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
    ILogicalOperator rightInputOp = rightInputOpRef.getValue();
    // Creates a variable to indicate whether a left input tuple is killed in the plan rooted at rightInputOp.
    LogicalVariable assignVar = context.newVar();
    ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<>(ConstantExpression.TRUE));
    assignOp.getInputs().add(rightInputOpRef);
    context.computeAndSetTypeEnvironmentForOperator(assignOp);
    rightInputOpRef = new MutableObject<>(assignOp);
    // Constructs the join predicate for the leftOuter join.
    List<Mutable<ILogicalExpression>> joinPredicates = new ArrayList<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        List<Mutable<ILogicalExpression>> arguments = new ArrayList<>();
        arguments.add(new MutableObject<>(new VariableReferenceExpression(liveVar)));
        LogicalVariable rightVar = varMap.get(liveVar);
        arguments.add(new MutableObject<>(new VariableReferenceExpression(rightVar)));
        ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), arguments);
        joinPredicates.add(new MutableObject<>(expr));
    }
    ILogicalExpression joinExpr = joinPredicates.size() > 1 ? new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), joinPredicates) : joinPredicates.size() > 0 ? joinPredicates.get(0).getValue() : ConstantExpression.TRUE;
    LeftOuterJoinOperator leftOuterJoinOp = new LeftOuterJoinOperator(new MutableObject<>(joinExpr), inputOpRef, rightInputOpRef);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(rightInputOp, context);
    context.computeAndSetTypeEnvironmentForOperator(leftOuterJoinOp);
    // Creates group-by operator.
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<ILogicalPlan> nestedPlans = new ArrayList<>();
    GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, nestedPlans);
    LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        LogicalVariable newVar = context.newVar();
        groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(liveVar))));
        // Adds variables for replacements in ancestors.
        replacedVarMap.put(liveVar, newVar);
    }
    for (LogicalVariable liveVar : inputLiveVars) {
        if (primaryKeyVars.contains(liveVar)) {
            continue;
        }
        groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
    }
    // Sets up the nested plan for the groupby operator.
    Mutable<ILogicalOperator> aggOpRef = subplanOp.getNestedPlans().get(0).getRoots().get(0);
    // Clears the input of the lowest aggregate.
    lowestAggregateRefInSubplan.getValue().getInputs().clear();
    Mutable<ILogicalOperator> currentOpRef = lowestAggregateRefInSubplan;
    // Adds an optional order operator.
    List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = varMapAndOrderExprs.second;
    if (!orderExprs.isEmpty()) {
        OrderOperator orderOp = new OrderOperator(orderExprs);
        currentOpRef = new MutableObject<>(orderOp);
        lowestAggregateRefInSubplan.getValue().getInputs().add(currentOpRef);
    }
    // Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
    // subplan input tuples that are filtered out within a subplan.
    Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(assignVar));
    List<Mutable<ILogicalExpression>> args = new ArrayList<>();
    args.add(filterVarExpr);
    List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
    argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
    SelectOperator selectOp = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)), false, null);
    currentOpRef.getValue().getInputs().add(new MutableObject<>(selectOp));
    selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
    List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<>();
    nestedRoots.add(aggOpRef);
    nestedPlans.add(new ALogicalPlanImpl(nestedRoots));
    groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
    // Replaces subplan with the group-by operator.
    opRef.setValue(groupbyOp);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
    // Recursively applys this rule to the nested plan of the subplan operator,
    // for the case where there are nested subplan operators within {@code subplanOp}.
    Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(rightInputOpRef, context);
    VariableUtilities.substituteVariables(leftOuterJoinOp, result.second, context);
    VariableUtilities.substituteVariables(groupbyOp, result.second, context);
    // No var mapping from the right input operator should be populated up.
    return new Pair<>(true, replacedVarMap);
}

Example 8 with LeftOuterJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator in project asterixdb by apache.

the class AqlPlusExpressionToPlanTranslator method visitJoinClause.

@Override
public Pair<ILogicalOperator, LogicalVariable> visitJoinClause(JoinClause jc, Mutable<ILogicalOperator> tupSource) throws CompilationException {
    Mutable<ILogicalOperator> opRef = tupSource;
    Pair<ILogicalOperator, LogicalVariable> leftSide = null;
    for (Clause c : jc.getLeftClauses()) {
        leftSide = c.accept(this, opRef);
        opRef = new MutableObject<ILogicalOperator>(leftSide.first);
    }
    opRef = tupSource;
    Pair<ILogicalOperator, LogicalVariable> rightSide = null;
    for (Clause c : jc.getRightClauses()) {
        rightSide = c.accept(this, opRef);
        opRef = new MutableObject<ILogicalOperator>(rightSide.first);
    }
    Pair<ILogicalExpression, Mutable<ILogicalOperator>> whereCond = langExprToAlgExpression(jc.getWhereExpr(), tupSource);
    AbstractBinaryJoinOperator join;
    switch(jc.getKind()) {
        case INNER:
            join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(whereCond.first));
            break;
        case LEFT_OUTER:
            join = new LeftOuterJoinOperator(new MutableObject<ILogicalExpression>(whereCond.first));
            break;
        default:
            throw new CompilationException(ErrorCode.COMPILATION_AQLPLUS_NO_SUCH_JOIN_TYPE);
    }
    join.getInputs().add(new MutableObject<ILogicalOperator>(leftSide.first));
    join.getInputs().add(new MutableObject<ILogicalOperator>(rightSide.first));
    return new Pair<ILogicalOperator, LogicalVariable>(join, null);
}

Example 9 with LeftOuterJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator in project asterixdb by apache.

the class FuzzyJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    // current opperator is join
    if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    // Find GET_ITEM function.
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
    Mutable<ILogicalExpression> expRef = joinOp.getCondition();
    Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
    if (getItemExprRef == null) {
        return false;
    }
    // Check if the GET_ITEM function is on one of the supported similarity-check functions.
    AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
    Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
    AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
    if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
        return false;
    }
    // Skip this rule based on annotations.
    if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
        return false;
    }
    List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
    ILogicalOperator leftInputOp = inputOps.get(0).getValue();
    ILogicalOperator rightInputOp = inputOps.get(1).getValue();
    List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
    ILogicalExpression inputExp0 = inputExps.get(0).getValue();
    ILogicalExpression inputExp1 = inputExps.get(1).getValue();
    // left and right expressions are variables
    if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
    LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
    LogicalVariable leftInputVar;
    LogicalVariable rightInputVar;
    liveVars.clear();
    VariableUtilities.getLiveVariables(leftInputOp, liveVars);
    if (liveVars.contains(inputVar0)) {
        leftInputVar = inputVar0;
        rightInputVar = inputVar1;
    } else {
        leftInputVar = inputVar1;
        rightInputVar = inputVar0;
    }
    List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
    List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
    // Bail if primary keys could not be inferred.
    if (leftInputPKs == null || rightInputPKs == null) {
        return false;
    }
    // primary key has only one variable
    if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
        return false;
    }
    IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
    IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
    // left-hand side and right-hand side of "~=" has the same type
    IAType left2 = TypeComputeUtils.getActualType(leftType);
    IAType right2 = TypeComputeUtils.getActualType(rightType);
    if (!left2.deepEqual(right2)) {
        return false;
    }
    //
    // -- - FIRE - --
    //
    MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider());
    FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
    String tokenizer;
    if (funcId == null) {
        tokenizer = "";
    } else {
        tokenizer = funcId.getName();
    }
    float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
    String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
    // finalize AQL+ query
    String prepareJoin;
    switch(joinOp.getJoinKind()) {
        case INNER:
            {
                prepareJoin = "join" + AQLPLUS;
                break;
            }
        case LEFT_OUTER:
            {
                // other sort of bug.
                return false;
            // prepareJoin = "loj" + AQLPLUS;
            // break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold, tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
    LogicalVariable leftPKVar = leftInputPKs.get(0);
    LogicalVariable rightPKVar = rightInputPKs.get(0);
    Counter counter = new Counter(context.getVarCounter());
    AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
    parser.initScope();
    parser.setVarCounter(counter);
    List<Clause> clauses;
    try {
        clauses = parser.Clauses();
    } catch (ParseException e) {
        throw new AlgebricksException(e);
    }
    // The translator will compile metadata internally. Run this compilation
    // under the same transaction id as the "outer" compilation.
    AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(metadataProvider, counter);
    context.setVarCounter(counter.get());
    LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
    translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
    translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
    translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
    translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
    translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp));
    translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    //
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // TODO pick side to run Stage 1, currently always picks RIGHT side
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    ILogicalPlan plan;
    try {
        plan = translator.translate(clauses);
    } catch (AsterixException e) {
        throw new AlgebricksException(e);
    }
    context.setVarCounter(counter.get());
    ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
    SelectOperator extraSelect = null;
    if (getItemExprRef != expRef) {
        // more than one join condition
        getItemExprRef.setValue(ConstantExpression.TRUE);
        switch(joinOp.getJoinKind()) {
            case INNER:
                {
                    extraSelect = new SelectOperator(expRef, false, null);
                    extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
                    outputOp = extraSelect;
                    break;
                }
            case LEFT_OUTER:
                {
                    if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
                        throw new IllegalStateException();
                    }
                    LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
                    topJoin.getCondition().setValue(expRef.getValue());
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
    }
    opRef.setValue(outputOp);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    return true;
}

Example 10 with LeftOuterJoinOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator in project asterixdb by apache.

the class LogicalOperatorDeepCopyWithNewVariablesVisitor method visitLeftOuterJoinOperator.

@Override
public ILogicalOperator visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
    LeftOuterJoinOperator opCopy = new LeftOuterJoinOperator(exprDeepCopyVisitor.deepCopyExpressionReference(op.getCondition()), deepCopyOperatorReference(op.getInputs().get(0), arg), deepCopyOperatorReference(op.getInputs().get(1), arg));
    copyAnnotations(op, opCopy);
    opCopy.setExecutionMode(op.getExecutionMode());
    return opCopy;
}