Examples with UnnestOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator used on opensource projects

Example 16 with UnnestOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator in project asterixdb by apache.

the class IntroduceGroupByForSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op0;
    Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
    Mutable<ILogicalOperator> botRef = subplanRoot;
    AbstractLogicalOperator op2;
    // Project is optional
    if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
        op2 = op1;
    } else {
        ProjectOperator project = (ProjectOperator) op1;
        botRef = project.getInputs().get(0);
        op2 = (AbstractLogicalOperator) botRef.getValue();
    }
    if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggregate = (AggregateOperator) op2;
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    VariableUtilities.getUsedVariables(aggregate, free);
    Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
    AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
    while (op3.getInputs().size() == 1) {
        Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariables(op3, prod);
        free.removeAll(prod);
        VariableUtilities.getUsedVariables(op3, free);
        botRef = op3Ref;
        op3Ref = op3.getInputs().get(0);
        op3 = (AbstractLogicalOperator) op3Ref.getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
    if (join.getCondition().getValue() == ConstantExpression.TRUE) {
        return false;
    }
    VariableUtilities.getUsedVariables(join, free);
    AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
    // see if there's an NTS at the end of the pipeline
    NestedTupleSourceOperator outerNts = getNts(b0);
    if (outerNts == null) {
        AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
        outerNts = getNts(b1);
        if (outerNts == null) {
            return false;
        }
    }
    Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
    if (pkVars == null || pkVars.size() < 1) {
        // there is no non-trivial primary key, group-by keys are all live variables
        // that were produced by descendant or self
        ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
        pkVars = new HashSet<LogicalVariable>();
        //get live variables
        VariableUtilities.getLiveVariables(subplanInput, pkVars);
        //get produced variables
        Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
        //retain the intersection
        pkVars.retainAll(producedVars);
    }
    AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
    Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
    LogicalVariable testForNull = null;
    AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
    switch(right.getOperatorTag()) {
        case UNNEST:
            {
                UnnestOperator innerUnnest = (UnnestOperator) right;
                // Select [ $y != null ]
                testForNull = innerUnnest.getVariable();
                break;
            }
        case RUNNINGAGGREGATE:
            {
                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
                if (!producedVars.isEmpty()) {
                    // Select [ $y != null ]
                    testForNull = producedVars.iterator().next();
                }
                break;
            }
        case DATASOURCESCAN:
            {
                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
                // Select [ $y != null ]
                if (innerScan.getVariables().size() == 1) {
                    testForNull = innerScan.getVariables().get(0);
                }
                break;
            }
        default:
            break;
    }
    if (testForNull == null) {
        testForNull = context.newVar();
        AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
        tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
        rightRef.setValue(tmpAsgn);
        context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
    }
    IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
    ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
    IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
    ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
    SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
    GroupByOperator g = new GroupByOperator();
    Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
    NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
    opRef.setValue(g);
    selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
    List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
    prodInpList.clear();
    prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
    ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
    g.getNestedPlans().add(gPlan);
    subplanRoot.setValue(op3Ref.getValue());
    g.getInputs().add(newSubplanRef);
    HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
    underVars.removeAll(pkVars);
    Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
    context.updatePrimaryKeys(mappedVars);
    for (LogicalVariable uv : underVars) {
        g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
    }
    OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
    OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
    context.computeAndSetTypeEnvironmentForOperator(g);
    return true;
}

Example 17 with UnnestOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator in project asterixdb by apache.

the class IsomorphismOperatorVisitor method visitUnnestOperator.

@Override
public Boolean visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
    AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
    if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return Boolean.FALSE;
    }
    UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg);
    boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables()) && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable());
    if (!isomorphic) {
        return Boolean.FALSE;
    }
    isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
    return isomorphic;
}

Example 18 with UnnestOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator in project asterixdb by apache.

the class RemoveUnusedAssignAndAggregateRule method collectUnusedAssignedVars.

private void collectUnusedAssignedVars(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> accumulatedUsedVarFromRootSet, boolean first, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (!first) {
        context.addToDontApplySet(this, op);
    }
    Set<LogicalVariable> assignVarsSetInThisOp = new HashSet<>();
    Set<LogicalVariable> usedVarsSetInThisOp = new HashSet<>();
    // Add used variables in this operator to the accumulated used variables set?
    boolean addUsedVarsInThisOp = true;
    // ASSIGN, AGGREGATE, UNNEST, UNIONALL, or GROUP operator found?
    boolean targetOpFound = false;
    switch(op.getOperatorTag()) {
        case ASSIGN:
            AssignOperator assign = (AssignOperator) op;
            assignVarsSetInThisOp.addAll(assign.getVariables());
            targetOpFound = true;
            break;
        case AGGREGATE:
            AggregateOperator agg = (AggregateOperator) op;
            assignVarsSetInThisOp.addAll(agg.getVariables());
            targetOpFound = true;
            break;
        case UNNEST:
            UnnestOperator uOp = (UnnestOperator) op;
            LogicalVariable pVar = uOp.getPositionalVariable();
            if (pVar != null) {
                assignVarsSetInThisOp.add(pVar);
                targetOpFound = true;
            }
            break;
        case UNIONALL:
            UnionAllOperator unionOp = (UnionAllOperator) op;
            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : unionOp.getVariableMappings()) {
                assignVarsSetInThisOp.add(varMapping.third);
            }
            targetOpFound = true;
            // Don't add used variables in UNIONALL.
            addUsedVarsInThisOp = false;
            break;
        case GROUP:
            GroupByOperator groupByOp = (GroupByOperator) op;
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorMapping : groupByOp.getDecorList()) {
                LogicalVariable decorVar = decorMapping.first;
                if (decorVar != null) {
                    assignVarsSetInThisOp.add(decorVar);
                    targetOpFound = true;
                } else {
                    // A decor var mapping can have a variable reference expression without a new variable
                    // definition, which is for rebinding the referred variable.
                    VariableReferenceExpression varExpr = (VariableReferenceExpression) decorMapping.second.getValue();
                    LogicalVariable reboundDecorVar = varExpr.getVariableReference();
                    assignVarsSetInThisOp.add(reboundDecorVar);
                }
            }
            break;
        default:
            break;
    }
    if (targetOpFound) {
        assignedVarMap.put(opRef, assignVarsSetInThisOp);
        assignedVarSet.addAll(assignVarsSetInThisOp);
    }
    if (addUsedVarsInThisOp) {
        VariableUtilities.getUsedVariables(op, usedVarsSetInThisOp);
        accumulatedUsedVarFromRootSet.addAll(usedVarsSetInThisOp);
        // paths in the plan.
        if (accumulatedUsedVarFromRootMap.containsKey(opRef)) {
            accumulatedUsedVarFromRootMap.get(opRef).addAll(accumulatedUsedVarFromRootSet);
        } else {
            accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
        }
    } else {
        accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
    }
    for (Mutable<ILogicalOperator> c : op.getInputs()) {
        collectUnusedAssignedVars(c, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
                collectUnusedAssignedVars(r, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
            }
        }
    }
}

Example 19 with UnnestOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator in project asterixdb by apache.

the class ListifyUnnestingFunctionRule method listifyUnnestingFunction.

// Performs the actual logical transformation.
private boolean listifyUnnestingFunction(ILogicalOperator op, Mutable<ILogicalExpression> exprRef, AbstractFunctionCallExpression func, IOptimizationContext context) throws AlgebricksException {
    IFunctionInfo functionInfo = func.getFunctionInfo();
    // Checks if the function is an unnesting function.
    if (!BuiltinFunctions.isBuiltinUnnestingFunction(functionInfo.getFunctionIdentifier())) {
        return false;
    }
    // Generates the listified collection in a subplan.
    SubplanOperator subplanOperator = new SubplanOperator();
    // Creates a nested tuple source operator.
    NestedTupleSourceOperator ntsOperator = new NestedTupleSourceOperator(new MutableObject<>(subplanOperator));
    // Unnests the dataset.
    LogicalVariable unnestVar = context.newVar();
    ILogicalExpression unnestExpr = new UnnestingFunctionCallExpression(functionInfo, func.getArguments());
    UnnestOperator unnestOperator = new UnnestOperator(unnestVar, new MutableObject<>(unnestExpr));
    unnestOperator.getInputs().add(new MutableObject<>(ntsOperator));
    // Listify the dataset into one collection.
    LogicalVariable aggVar = context.newVar();
    Mutable<ILogicalExpression> aggArgExprRef = new MutableObject<>(new VariableReferenceExpression(unnestVar));
    ILogicalExpression aggExpr = new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, new ArrayList<>(Collections.singletonList(aggArgExprRef)));
    AggregateOperator aggregateOperator = new AggregateOperator(new ArrayList<>(Collections.singletonList(aggVar)), new ArrayList<>(Collections.singletonList(new MutableObject<>(aggExpr))));
    aggregateOperator.getInputs().add(new MutableObject<>(unnestOperator));
    // Adds the aggregate operator as the root of the subplan.
    subplanOperator.setRootOp(new MutableObject<>(aggregateOperator));
    // Sticks a subplan operator into the query plan.
    // Note: given the way we compile JOINs, the unnesting function expression cannot appear in
    // any binary operators.
    // Example test queries:
    // asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-2
    // asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-3
    subplanOperator.getInputs().add(op.getInputs().get(0));
    op.getInputs().set(0, new MutableObject<>(subplanOperator));
    exprRef.setValue(new VariableReferenceExpression(aggVar));
    // Computes type environments for new operators.
    context.computeAndSetTypeEnvironmentForOperator(ntsOperator);
    context.computeAndSetTypeEnvironmentForOperator(unnestOperator);
    context.computeAndSetTypeEnvironmentForOperator(aggregateOperator);
    context.computeAndSetTypeEnvironmentForOperator(subplanOperator);
    return true;
}

Example 20 with UnnestOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator in project asterixdb by apache.

the class PullPositionalVariableFromUnnestRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return false;
    }
    UnnestOperator unnest = (UnnestOperator) op;
    LogicalVariable p = unnest.getPositionalVariable();
    if (p == null) {
        return false;
    }
    ArrayList<LogicalVariable> rOpVars = new ArrayList<LogicalVariable>();
    rOpVars.add(p);
    ArrayList<Mutable<ILogicalExpression>> rOpExprList = new ArrayList<Mutable<ILogicalExpression>>();
    StatefulFunctionCallExpression fce = new StatefulFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.TID), UnpartitionedPropertyComputer.INSTANCE);
    rOpExprList.add(new MutableObject<ILogicalExpression>(fce));
    RunningAggregateOperator rOp = new RunningAggregateOperator(rOpVars, rOpExprList);
    rOp.setExecutionMode(unnest.getExecutionMode());
    RunningAggregatePOperator rPop = new RunningAggregatePOperator();
    rOp.setPhysicalOperator(rPop);
    rOp.getInputs().add(new MutableObject<ILogicalOperator>(unnest));
    opRef.setValue(rOp);
    unnest.setPositionalVariable(null);
    context.computeAndSetTypeEnvironmentForOperator(rOp);
    context.computeAndSetTypeEnvironmentForOperator(unnest);
    return true;
}