Examples with VariableReferenceExpression org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression used on opensource projects

Example 61 with VariableReferenceExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression in project asterixdb by apache.

the class IntroduceGroupByForSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op0;
    Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
    Mutable<ILogicalOperator> botRef = subplanRoot;
    AbstractLogicalOperator op2;
    // Project is optional
    if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
        op2 = op1;
    } else {
        ProjectOperator project = (ProjectOperator) op1;
        botRef = project.getInputs().get(0);
        op2 = (AbstractLogicalOperator) botRef.getValue();
    }
    if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggregate = (AggregateOperator) op2;
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    VariableUtilities.getUsedVariables(aggregate, free);
    Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
    AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
    while (op3.getInputs().size() == 1) {
        Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariables(op3, prod);
        free.removeAll(prod);
        VariableUtilities.getUsedVariables(op3, free);
        botRef = op3Ref;
        op3Ref = op3.getInputs().get(0);
        op3 = (AbstractLogicalOperator) op3Ref.getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
    if (join.getCondition().getValue() == ConstantExpression.TRUE) {
        return false;
    }
    VariableUtilities.getUsedVariables(join, free);
    AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
    // see if there's an NTS at the end of the pipeline
    NestedTupleSourceOperator outerNts = getNts(b0);
    if (outerNts == null) {
        AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
        outerNts = getNts(b1);
        if (outerNts == null) {
            return false;
        }
    }
    Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
    if (pkVars == null || pkVars.size() < 1) {
        // there is no non-trivial primary key, group-by keys are all live variables
        // that were produced by descendant or self
        ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
        pkVars = new HashSet<LogicalVariable>();
        //get live variables
        VariableUtilities.getLiveVariables(subplanInput, pkVars);
        //get produced variables
        Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
        //retain the intersection
        pkVars.retainAll(producedVars);
    }
    AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
    Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
    LogicalVariable testForNull = null;
    AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
    switch(right.getOperatorTag()) {
        case UNNEST:
            {
                UnnestOperator innerUnnest = (UnnestOperator) right;
                // Select [ $y != null ]
                testForNull = innerUnnest.getVariable();
                break;
            }
        case RUNNINGAGGREGATE:
            {
                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
                if (!producedVars.isEmpty()) {
                    // Select [ $y != null ]
                    testForNull = producedVars.iterator().next();
                }
                break;
            }
        case DATASOURCESCAN:
            {
                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
                // Select [ $y != null ]
                if (innerScan.getVariables().size() == 1) {
                    testForNull = innerScan.getVariables().get(0);
                }
                break;
            }
        default:
            break;
    }
    if (testForNull == null) {
        testForNull = context.newVar();
        AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
        tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
        rightRef.setValue(tmpAsgn);
        context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
    }
    IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
    ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
    IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
    ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
    SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
    GroupByOperator g = new GroupByOperator();
    Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
    NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
    opRef.setValue(g);
    selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
    List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
    prodInpList.clear();
    prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
    ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
    g.getNestedPlans().add(gPlan);
    subplanRoot.setValue(op3Ref.getValue());
    g.getInputs().add(newSubplanRef);
    HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
    underVars.removeAll(pkVars);
    Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
    context.updatePrimaryKeys(mappedVars);
    for (LogicalVariable uv : underVars) {
        g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
    }
    OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
    OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
    context.computeAndSetTypeEnvironmentForOperator(g);
    return true;
}

Example 62 with VariableReferenceExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression in project asterixdb by apache.

the class LogicalExpressionDeepCopyWithNewVariablesVisitor method visitVariableReferenceExpression.

@Override
public ILogicalExpression visitVariableReferenceExpression(VariableReferenceExpression expr, Void arg) throws AlgebricksException {
    LogicalVariable var = expr.getVariableReference();
    LogicalVariable givenVarReplacement = inVarMapping.get(var);
    if (givenVarReplacement != null) {
        outVarMapping.put(var, givenVarReplacement);
        return new VariableReferenceExpression(givenVarReplacement);
    }
    LogicalVariable varCopy = outVarMapping.get(var);
    if (varCopy == null) {
        varCopy = varContext.newVar();
        outVarMapping.put(var, varCopy);
    }
    return new VariableReferenceExpression(varCopy);
}

Example 63 with VariableReferenceExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression in project asterixdb by apache.

the class RemoveRedundantVariablesRule method removeRedundantVariables.

private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    LogicalOperatorTag opTag = op.getOperatorTag();
    boolean modified = false;
    // Update equivalence class map.
    if (opTag == LogicalOperatorTag.ASSIGN) {
        AssignOperator assignOp = (AssignOperator) op;
        int numVars = assignOp.getVariables().size();
        for (int i = 0; i < numVars; i++) {
            ILogicalExpression expr = assignOp.getExpressions().get(i).getValue();
            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                continue;
            }
            VariableReferenceExpression rhsVarRefExpr = (VariableReferenceExpression) expr;
            // Update equivalence class map.
            LogicalVariable lhs = assignOp.getVariables().get(i);
            LogicalVariable rhs = rhsVarRefExpr.getVariableReference();
            updateEquivalenceClassMap(lhs, rhs);
        }
    }
    // Replace variable references with their first representative.
    if (opTag == LogicalOperatorTag.PROJECT) {
        // The project operator does not use expressions, so we need to replace it's variables manually.
        if (replaceProjectVars((ProjectOperator) op)) {
            modified = true;
        }
    } else if (op.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
        // Replace redundant variables manually in the UnionAll operator.
        if (replaceUnionAllVars((UnionAllOperator) op)) {
            modified = true;
        }
    } else {
        if (op.acceptExpressionTransform(substVisitor)) {
            modified = true;
        }
    }
    // Perform variable replacement in nested plans.
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
            for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
                if (removeRedundantVariables(rootRef, context)) {
                    modified = true;
                }
            }
        }
    }
    // Deal with re-mapping of variables in group by.
    if (opTag == LogicalOperatorTag.GROUP) {
        if (handleGroupByVarRemapping((GroupByOperator) op)) {
            modified = true;
        }
    }
    if (modified) {
        context.computeAndSetTypeEnvironmentForOperator(op);
        context.addToDontApplySet(this, op);
    }
    // in the query plan.
    if (opTag == LogicalOperatorTag.DISTRIBUTE_RESULT || opTag == LogicalOperatorTag.SINK) {
        equivalentVarsMap.clear();
    }
    return modified;
}

Example 64 with VariableReferenceExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression in project asterixdb by apache.

the class RemoveRedundantVariablesRule method handleGroupByVarRemapping.

private boolean handleGroupByVarRemapping(GroupByOperator groupOp) {
    boolean modified = false;
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gp : groupOp.getGroupByList()) {
        if (gp.first == null || gp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
            continue;
        }
        LogicalVariable groupByVar = ((VariableReferenceExpression) gp.second.getValue()).getVariableReference();
        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
        while (iter.hasNext()) {
            Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
            if (dp.first != null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                continue;
            }
            LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
            if (dv == groupByVar) {
                // The decor variable is redundant, since it is propagated as a grouping variable.
                List<LogicalVariable> equivalentVars = equivalentVarsMap.get(groupByVar);
                if (equivalentVars != null) {
                    // Change representative of this equivalence class.
                    equivalentVars.set(0, gp.first);
                    equivalentVarsMap.put(gp.first, equivalentVars);
                } else {
                    updateEquivalenceClassMap(gp.first, groupByVar);
                }
                iter.remove();
                modified = true;
                break;
            }
        }
    }
    // find the redundant variables within the decor list
    Map<LogicalVariable, LogicalVariable> variableToFirstDecorMap = new HashMap<LogicalVariable, LogicalVariable>();
    Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
    while (iter.hasNext()) {
        Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
        if (dp.first == null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
            continue;
        }
        LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
        LogicalVariable firstDecor = variableToFirstDecorMap.get(dv);
        if (firstDecor == null) {
            variableToFirstDecorMap.put(dv, dp.first);
        } else {
            // The decor variable dp.first is redundant since firstDecor is exactly the same.
            updateEquivalenceClassMap(dp.first, firstDecor);
            iter.remove();
            modified = true;
        }
    }
    return modified;
}

Example 65 with VariableReferenceExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression in project asterixdb by apache.

the class RemoveUnusedAssignAndAggregateRule method removeUnusedVarsFromGroupBy.

private boolean removeUnusedVarsFromGroupBy(GroupByOperator groupByOp, Set<LogicalVariable> toRemove) {
    if (toRemove == null || toRemove.isEmpty()) {
        return false;
    }
    Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupByOp.getDecorList().iterator();
    boolean modified = false;
    while (iter.hasNext()) {
        Pair<LogicalVariable, Mutable<ILogicalExpression>> varMapping = iter.next();
        LogicalVariable decorVar = varMapping.first;
        // A decor var mapping can have a variable reference expression without a new variable definition,
        // which is for rebinding the referred variable.
        VariableReferenceExpression varExpr = (VariableReferenceExpression) varMapping.second.getValue();
        LogicalVariable decorReferredVar = varExpr.getVariableReference();
        boolean removeReBoundDecorVar = toRemove.contains(decorReferredVar);
        if ((decorVar != null && toRemove.contains(decorVar)) || removeReBoundDecorVar) {
            iter.remove();
            modified = true;
            if (removeReBoundDecorVar) {
                // Do not need to remove that in the children pipeline.
                toRemove.remove(decorReferredVar);
            }
        }
    }
    return modified;
}