Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 6 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method removeRedundantListifies.

private void removeRedundantListifies(Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex) throws AlgebricksException {
    List<Pair<AbstractOperatorWithNestedPlans, Integer>> removeList = new ArrayList<>();
    for (Map.Entry<LogicalVariable, Integer> aggVarEntry : nspAggVars.entrySet()) {
        LogicalVariable aggVar = aggVarEntry.getKey();
        int occurs = aggVarEntry.getValue();
        if (occurs == 0) {
            AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(aggVar);
            AggregateOperator aggOp = (AggregateOperator) nspOp.getNestedPlans().get(nspAggVarToPlanIndex.get(aggVar)).getRoots().get(0).getValue();
            int pos = aggOp.getVariables().indexOf(aggVar);
            if (pos >= 0) {
                aggOp.getVariables().remove(pos);
                aggOp.getExpressions().remove(pos);
                List<LogicalVariable> producedVarsAtAgg = new ArrayList<>();
                VariableUtilities.getProducedVariablesInDescendantsAndSelf(aggOp, producedVarsAtAgg);
                if (producedVarsAtAgg.isEmpty()) {
                    removeList.add(new Pair<>(nspOp, nspAggVarToPlanIndex.get(aggVar)));
                }
            }
        }
    }
    // Collects subplans that is to be removed.
    Map<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> nspToSubplanListMap = new HashMap<>();
    for (Pair<AbstractOperatorWithNestedPlans, Integer> remove : removeList) {
        AbstractOperatorWithNestedPlans groupByOperator = remove.first;
        ILogicalPlan subplan = remove.first.getNestedPlans().get(remove.second);
        if (nspToSubplanListMap.containsKey(groupByOperator)) {
            List<ILogicalPlan> subplans = nspToSubplanListMap.get(groupByOperator);
            subplans.add(subplan);
        } else {
            List<ILogicalPlan> subplans = new ArrayList<>();
            subplans.add(subplan);
            nspToSubplanListMap.put(groupByOperator, subplans);
        }
    }
    // Removes subplans.
    for (Map.Entry<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> entry : nspToSubplanListMap.entrySet()) {
        entry.getKey().getNestedPlans().removeAll(entry.getValue());
    }
}

Example 7 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method pushSubplanAsAggIntoNestedSubplan.

private boolean pushSubplanAsAggIntoNestedSubplan(Mutable<ILogicalOperator> subplanOpRef, AbstractOperatorWithNestedPlans nspOp, LogicalVariable varFromNestedAgg, Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
    // only free var can be varFromNestedAgg
    HashSet<LogicalVariable> freeVars = new HashSet<>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
    for (LogicalVariable vFree : freeVars) {
        if (!vFree.equals(varFromNestedAgg)) {
            return false;
        }
    }
    List<ILogicalPlan> plans = subplan.getNestedPlans();
    if (plans.size() > 1) {
        return false;
    }
    ILogicalPlan p = plans.get(0);
    if (p.getRoots().size() > 1) {
        return false;
    }
    Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggInSubplanOp = (AggregateOperator) op;
    LogicalVariable unnestVar = null;
    boolean pushableNestedSubplan = false;
    while (op.getInputs().size() == 1) {
        opRef = op.getInputs().get(0);
        op = (AbstractLogicalOperator) opRef.getValue();
        switch(op.getOperatorTag()) {
            case ASSIGN:
                break;
            case UNNEST:
                UnnestOperator unnest = (UnnestOperator) op;
                if (unnest.getPositionalVariable() != null) {
                    // TODO currently subplan with both accumulating and running aggregate is not supported.
                    return false;
                }
                ILogicalExpression expr = unnest.getExpressionRef().getValue();
                if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
                    return false;
                }
                AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
                if (fun.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
                    return false;
                }
                ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
                if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                    return false;
                }
                VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
                if (!varExpr.getVariableReference().equals(varFromNestedAgg)) {
                    return false;
                }
                opRef = op.getInputs().get(0);
                op = (AbstractLogicalOperator) opRef.getValue();
                if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                    return false;
                }
                pushableNestedSubplan = true;
                unnestVar = unnest.getVariable();
                break;
            default:
                return false;
        }
    }
    if (!pushableNestedSubplan) {
        return false;
    }
    for (int i = 0; i < nspOp.getNestedPlans().size(); i++) {
        Mutable<ILogicalOperator> nspAggRef = nspOp.getNestedPlans().get(i).getRoots().get(0);
        AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue();
        Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0);
        LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg);
        if (listifyVar == null) {
            continue;
        }
        OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context);
        nspAgg.getVariables().addAll(aggInSubplanOp.getVariables());
        nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
        for (LogicalVariable v : aggInSubplanOp.getVariables()) {
            nspWithAgg.put(v, nspOp);
            nspAggVars.put(v, 0);
            nspAggVarToPlanIndex.put(v, i);
        }
        Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
        if (!opRef1InSubplan.getValue().getInputs().isEmpty()) {
            Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
            AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
            if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs();
                nspInpList.clear();
                nspInpList.add(opRef1InSubplan);
                while (true) {
                    opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
                    op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
                    if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
                        List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
                        opInpList.clear();
                        opInpList.add(nspAggChildRef);
                        break;
                    }
                    opRef1InSubplan = opRef2InSubplan;
                    if (opRef1InSubplan.getValue().getInputs().isEmpty()) {
                        throw new IllegalStateException("PushAggregateIntoNestedSubplanRule: could not find UNNEST.");
                    }
                }
            }
        }
        subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
        OperatorPropertiesUtil.typeOpRec(nspAggRef, context);
    }
    return true;
}

Example 8 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushFieldAccessRule method propagateFieldAccessRec.

@SuppressWarnings("unchecked")
private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context, String finalAnnot) throws AlgebricksException {
    AssignOperator access = (AssignOperator) opRef.getValue();
    Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    // rewritten into index search.
    if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2) && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
        return false;
    }
    Object annotation = op2.getAnnotations().get(IS_MOVABLE);
    if (annotation != null && !((Boolean) annotation)) {
        return false;
    }
    if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
        return false;
    }
    if (testAndModifyRedundantOp(access, op2)) {
        propagateFieldAccessRec(opRef2, context, finalAnnot);
        return true;
    }
    List<LogicalVariable> usedInAccess = new LinkedList<>();
    VariableUtilities.getUsedVariables(access, usedInAccess);
    List<LogicalVariable> produced2 = new LinkedList<>();
    if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        VariableUtilities.getLiveVariables(op2, produced2);
    } else {
        VariableUtilities.getProducedVariables(op2, produced2);
    }
    boolean pushItDown = false;
    List<LogicalVariable> inter = new ArrayList<>(usedInAccess);
    if (inter.isEmpty()) {
        // ground value
        return false;
    }
    inter.retainAll(produced2);
    if (inter.isEmpty()) {
        pushItDown = true;
    } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        GroupByOperator g = (GroupByOperator) op2;
        List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
            ILogicalExpression e = p.second.getValue();
            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
                if (inter.contains(decorVar)) {
                    inter.remove(decorVar);
                    LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
                    varMappings.add(new Pair<>(decorVar, v1));
                }
            }
        }
        if (inter.isEmpty()) {
            boolean changed = false;
            for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
                LogicalVariable v2 = context.newVar();
                LogicalVariable oldVar = access.getVariables().get(0);
                g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
                changed = true;
                access.getVariables().set(0, v2);
                VariableUtilities.substituteVariables(access, m.first, m.second, context);
            }
            if (changed) {
                context.computeAndSetTypeEnvironmentForOperator(g);
            }
            usedInAccess.clear();
            VariableUtilities.getUsedVariables(access, usedInAccess);
            pushItDown = true;
        }
    }
    if (pushItDown) {
        if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference().getValue().getInputs().get(0);
            pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
            return true;
        }
        if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
            pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
            return true;
        } else {
            for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
                HashSet<LogicalVariable> v2 = new HashSet<>();
                VariableUtilities.getLiveVariables(inp.getValue(), v2);
                if (v2.containsAll(usedInAccess)) {
                    pushAccessDown(opRef, op2, inp, context, finalAnnot);
                    return true;
                }
            }
        }
        if (op2.hasNestedPlans()) {
            AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
            for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
                for (Mutable<ILogicalOperator> root : plan.getRoots()) {
                    HashSet<LogicalVariable> v2 = new HashSet<>();
                    VariableUtilities.getLiveVariables(root.getValue(), v2);
                    if (v2.containsAll(usedInAccess)) {
                        pushAccessDown(opRef, op2, root, context, finalAnnot);
                        return true;
                    }
                }
            }
        }
        throw new AlgebricksException("Field access " + access.getExpressions().get(0).getValue() + " does not correspond to any input of operator " + op2);
    } else {
        // fields. If yes, we can equate the two variables.
        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator scan = (DataSourceScanOperator) op2;
            int n = scan.getVariables().size();
            LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
            AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions().get(0).getValue();
            ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
            LogicalExpressionTag tag = e0.getExpressionTag();
            if (tag == LogicalExpressionTag.VARIABLE) {
                VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
                if (varRef.getVariableReference() == scanRecordVar) {
                    ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
                    if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
                        IDataSource<DataSourceId> dataSource = (IDataSource<DataSourceId>) scan.getDataSource();
                        byte dsType = ((DataSource) dataSource).getDatasourceType();
                        if (dsType == DataSource.Type.FEED || dsType == DataSource.Type.LOADABLE) {
                            return false;
                        }
                        DataSourceId asid = dataSource.getId();
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
                        if (dataset == null) {
                            throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
                        }
                        if (dataset.getDatasetType() != DatasetType.INTERNAL) {
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        ConstantExpression ce = (ConstantExpression) e1;
                        IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
                        String fldName;
                        if (obj.getType().getTypeTag() == ATypeTag.STRING) {
                            fldName = ((AString) obj).getStringValue();
                        } else {
                            int pos = ((AInt32) obj).getIntegerValue();
                            String tName = dataset.getItemTypeName();
                            IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
                            if (t.getTypeTag() != ATypeTag.OBJECT) {
                                return false;
                            }
                            ARecordType rt = (ARecordType) t;
                            if (pos >= rt.getFieldNames().length) {
                                setAsFinal(access, context, finalAnnot);
                                return false;
                            }
                            fldName = rt.getFieldNames()[pos];
                        }
                        int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName);
                        if (p < 0) {
                            // not one of the partitioning fields
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        LogicalVariable keyVar = scan.getVariables().get(p);
                        access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
                        return true;
                    }
                }
            }
        }
        setAsFinal(access, context, finalAnnot);
        return false;
    }
}

Example 9 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class RemoveLeftOuterUnnestForLeftOuterJoinRule method checkNestedPlan.

// Checks the nested plan for the group-by operator.
private Pair<Boolean, ILogicalOperator> checkNestedPlan(GroupByOperator gbyOperator) {
    List<ILogicalPlan> nestedPlans = gbyOperator.getNestedPlans();
    if (nestedPlans.size() > 1) {
        return new Pair<>(false, null);
    }
    ILogicalPlan plan = nestedPlans.get(0);
    List<Mutable<ILogicalOperator>> roots = plan.getRoots();
    if (roots.size() > 1) {
        return new Pair<>(false, null);
    }
    ILogicalOperator root = roots.get(0).getValue();
    return new Pair<>(true, root);
}

Example 10 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushGroupByThroughProduct method canPushThrough.

private PushTestResult canPushThrough(GroupByOperator gby, ILogicalOperator branch, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> toPush, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> notToPush) throws AlgebricksException {
    Collection<LogicalVariable> fromBranch = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(branch, fromBranch);
    Collection<LogicalVariable> usedInGbyExprList = new ArrayList<LogicalVariable>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) {
        p.second.getValue().getUsedVariables(usedInGbyExprList);
    }
    if (!fromBranch.containsAll(usedInGbyExprList)) {
        return PushTestResult.FALSE;
    }
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    for (ILogicalPlan p : gby.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), free);
        }
    }
    if (!fromBranch.containsAll(free)) {
        return PushTestResult.FALSE;
    }
    Set<LogicalVariable> decorVarRhs = new HashSet<LogicalVariable>();
    decorVarRhs.clear();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
        ILogicalExpression expr = p.second.getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
            return PushTestResult.FALSE;
        }
        VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
        LogicalVariable v = varRef.getVariableReference();
        if (decorVarRhs.contains(v)) {
            return PushTestResult.REPEATED_DECORS;
        }
        decorVarRhs.add(v);
        if (fromBranch.contains(v)) {
            toPush.add(p);
        } else {
            notToPush.add(p);
        }
    }
    return PushTestResult.TRUE;
}