Examples with SelectOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator used on opensource projects

Example 21 with SelectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator in project asterixdb by apache.

the class FuzzyJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    // current opperator is join
    if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    // Find GET_ITEM function.
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
    Mutable<ILogicalExpression> expRef = joinOp.getCondition();
    Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
    if (getItemExprRef == null) {
        return false;
    }
    // Check if the GET_ITEM function is on one of the supported similarity-check functions.
    AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
    Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
    AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
    if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
        return false;
    }
    // Skip this rule based on annotations.
    if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
        return false;
    }
    List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
    ILogicalOperator leftInputOp = inputOps.get(0).getValue();
    ILogicalOperator rightInputOp = inputOps.get(1).getValue();
    List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
    ILogicalExpression inputExp0 = inputExps.get(0).getValue();
    ILogicalExpression inputExp1 = inputExps.get(1).getValue();
    // left and right expressions are variables
    if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
    LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
    LogicalVariable leftInputVar;
    LogicalVariable rightInputVar;
    liveVars.clear();
    VariableUtilities.getLiveVariables(leftInputOp, liveVars);
    if (liveVars.contains(inputVar0)) {
        leftInputVar = inputVar0;
        rightInputVar = inputVar1;
    } else {
        leftInputVar = inputVar1;
        rightInputVar = inputVar0;
    }
    List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
    List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
    // Bail if primary keys could not be inferred.
    if (leftInputPKs == null || rightInputPKs == null) {
        return false;
    }
    // primary key has only one variable
    if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
        return false;
    }
    IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
    IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
    // left-hand side and right-hand side of "~=" has the same type
    IAType left2 = TypeComputeUtils.getActualType(leftType);
    IAType right2 = TypeComputeUtils.getActualType(rightType);
    if (!left2.deepEqual(right2)) {
        return false;
    }
    //
    // -- - FIRE - --
    //
    MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider());
    FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
    String tokenizer;
    if (funcId == null) {
        tokenizer = "";
    } else {
        tokenizer = funcId.getName();
    }
    float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
    String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
    // finalize AQL+ query
    String prepareJoin;
    switch(joinOp.getJoinKind()) {
        case INNER:
            {
                prepareJoin = "join" + AQLPLUS;
                break;
            }
        case LEFT_OUTER:
            {
                // other sort of bug.
                return false;
            // prepareJoin = "loj" + AQLPLUS;
            // break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold, tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
    LogicalVariable leftPKVar = leftInputPKs.get(0);
    LogicalVariable rightPKVar = rightInputPKs.get(0);
    Counter counter = new Counter(context.getVarCounter());
    AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
    parser.initScope();
    parser.setVarCounter(counter);
    List<Clause> clauses;
    try {
        clauses = parser.Clauses();
    } catch (ParseException e) {
        throw new AlgebricksException(e);
    }
    // The translator will compile metadata internally. Run this compilation
    // under the same transaction id as the "outer" compilation.
    AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(metadataProvider, counter);
    context.setVarCounter(counter.get());
    LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
    translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
    translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
    translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
    translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
    translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp));
    translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    //
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // TODO pick side to run Stage 1, currently always picks RIGHT side
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    ILogicalPlan plan;
    try {
        plan = translator.translate(clauses);
    } catch (AsterixException e) {
        throw new AlgebricksException(e);
    }
    context.setVarCounter(counter.get());
    ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
    SelectOperator extraSelect = null;
    if (getItemExprRef != expRef) {
        // more than one join condition
        getItemExprRef.setValue(ConstantExpression.TRUE);
        switch(joinOp.getJoinKind()) {
            case INNER:
                {
                    extraSelect = new SelectOperator(expRef, false, null);
                    extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
                    outputOp = extraSelect;
                    break;
                }
            case LEFT_OUTER:
                {
                    if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
                        throw new IllegalStateException();
                    }
                    LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
                    topJoin.getCondition().setValue(expRef.getValue());
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
    }
    opRef.setValue(outputOp);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    return true;
}

Example 22 with SelectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator in project asterixdb by apache.

the class LoadRecordFieldsRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (context.checkIfInDontApplySet(this, op1)) {
        return false;
    }
    if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
        AssignOperator a1 = (AssignOperator) op1;
        ILogicalExpression expr = getFirstExpr(a1);
        if (AnalysisUtil.isAccessToFieldRecord(expr)) {
            boolean res = findAndEliminateRedundantFieldAccess(a1, context);
            context.addToDontApplySet(this, op1);
            return res;
        }
    }
    exprVisitor.setTopOp(op1);
    exprVisitor.setContext(context);
    boolean res = op1.acceptExpressionTransform(exprVisitor);
    if (!res) {
        context.addToDontApplySet(this, op1);
    }
    if (res && op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
        // checking if we can annotate a Selection as using just one field
        // access
        SelectOperator sigma = (SelectOperator) op1;
        LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
        VariableUtilities.getUsedVariables(sigma, vars);
        if (vars.size() == 1) {
            // we can annotate Selection
            AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
            AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
            if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
                // f should be a call to a field/data access kind of
                // function
                sigma.getAnnotations().put(OperatorAnnotation.FIELD_ACCESS, f.getArguments().get(0));
            }
        }
    }
    // TODO: avoid having to recompute type env. here
    if (res) {
        OperatorPropertiesUtil.typeOpRec(opRef, context);
    }
    return res;
}

Example 23 with SelectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method collectVarsBottomUp.

private boolean collectVarsBottomUp(Mutable<ILogicalOperator> opRef, IOptimizationContext context, Map<LogicalVariable, Integer> nspListifyVarsCount, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex, Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    context.addToDontApplySet(this, op1);
    boolean change = false;
    for (Mutable<ILogicalOperator> child : op1.getInputs()) {
        if (collectVarsBottomUp(child, context, nspListifyVarsCount, nspWithAgg, nspAggVarToPlanIndex, aggregateExprToVarExpr)) {
            change = true;
        }
    }
    Set<LogicalVariable> used = new HashSet<>();
    VariableUtilities.getUsedVariables(op1, used);
    switch(op1.getOperatorTag()) {
        case ASSIGN:
        case SELECT:
            boolean found = false;
            // Do some prefiltering: check if the Assign uses any nsp vars.
            for (LogicalVariable v : used) {
                if (nspListifyVarsCount.get(v) != null) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                break;
            }
            if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
                AssignOperator assign = (AssignOperator) op1;
                for (Mutable<ILogicalExpression> exprRef : assign.getExpressions()) {
                    Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef, nspWithAgg, aggregateExprToVarExpr, context);
                    if (p.first) {
                        change = true;
                        exprRef.setValue(p.second);
                    }
                }
            }
            if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
                SelectOperator select = (SelectOperator) op1;
                Mutable<ILogicalExpression> exprRef = select.getCondition();
                Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef, nspWithAgg, aggregateExprToVarExpr, context);
                if (p.first) {
                    change = true;
                    exprRef.setValue(p.second);
                }
            }
            used.clear();
            VariableUtilities.getUsedVariables(op1, used);
            // increment the count for the ones which are still used
            for (LogicalVariable v : used) {
                Integer m = nspListifyVarsCount.get(v);
                if (m != null) {
                    nspListifyVarsCount.put(v, m + 1);
                }
            }
            break;
        case SUBPLAN:
            // Try to push the subplan into a group-by operator if possible.
            for (LogicalVariable v : used) {
                Integer m = nspListifyVarsCount.get(v);
                if (m != null) {
                    AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(v);
                    if (pushSubplanAsAggIntoNestedSubplan(opRef, nspOp, v, nspListifyVarsCount, nspWithAgg, nspAggVarToPlanIndex, context)) {
                        change = true;
                    } else {
                        nspListifyVarsCount.put(v, m + 1);
                    }
                }
            }
            if (!change) {
                // Collect aggregate variables for pushing aggregates into the subplan (if possible).
                collectAggregateVars(nspListifyVarsCount, nspWithAgg, nspAggVarToPlanIndex, (AbstractOperatorWithNestedPlans) op1);
            }
            break;
        case GROUP:
            // Collect aggregate variables for pushing aggregates into the nested subplan
            // of the group by operator (if possible).
            collectAggregateVars(nspListifyVarsCount, nspWithAgg, nspAggVarToPlanIndex, (AbstractOperatorWithNestedPlans) op1);
            break;
        default:
            for (LogicalVariable v : used) {
                Integer m = nspListifyVarsCount.get(v);
                if (m != null) {
                    nspListifyVarsCount.put(v, m + 1);
                }
            }
    }
    return change;
}

Example 24 with SelectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator in project asterixdb by apache.

the class SimilarityCheckRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    // Look for select.
    if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
        return false;
    }
    SelectOperator select = (SelectOperator) op;
    Mutable<ILogicalExpression> condExpr = select.getCondition();
    // Gather assigns below this select.
    List<AssignOperator> assigns = new ArrayList<AssignOperator>();
    AbstractLogicalOperator childOp = (AbstractLogicalOperator) select.getInputs().get(0).getValue();
    // Skip selects.
    while (childOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
        childOp = (AbstractLogicalOperator) childOp.getInputs().get(0).getValue();
    }
    while (childOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
        assigns.add((AssignOperator) childOp);
        childOp = (AbstractLogicalOperator) childOp.getInputs().get(0).getValue();
    }
    return replaceSelectConditionExprs(condExpr, assigns, context);
}

Example 25 with SelectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator in project asterixdb by apache.

the class BTreeAccessMethod method applyJoinPlanTransformation.

@Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
    Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
    // Determine if the index is applicable on the left or right side
    // (if both, we arbitrarily prefer the left side).
    Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex);
    OptimizableOperatorSubTree indexSubTree;
    OptimizableOperatorSubTree probeSubTree;
    // The following is just a sanity check.
    if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
        indexSubTree = rightSubTree;
        probeSubTree = leftSubTree;
    } else {
        return false;
    }
    LogicalVariable newNullPlaceHolderVar = null;
    if (isLeftOuterJoin) {
        //get a new null place holder variable that is the first field variable of the primary key
        //from the indexSubTree's datasourceScanOp
        newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
    }
    ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context);
    if (primaryIndexUnnestOp == null) {
        return false;
    }
    if (isLeftOuterJoin && hasGroupBy) {
        //reset the null place holder variable
        AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
    }
    // If there are conditions left, add a new select operator on top.
    indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
    if (conditionRef.getValue() != null) {
        SelectOperator topSelect = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar);
        topSelect.getInputs().add(indexSubTree.getRootRef());
        topSelect.setExecutionMode(ExecutionMode.LOCAL);
        context.computeAndSetTypeEnvironmentForOperator(topSelect);
        // Replace the original join with the new subtree rooted at the select op.
        joinRef.setValue(topSelect);
    } else {
        joinRef.setValue(indexSubTree.getRootRef().getValue());
    }
    return true;
}