Examples with AbstractLogicalOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator used on opensource projects

Example 71 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method getDataset.

private Dataset getDataset(AbstractLogicalOperator op, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    while (descendantOp != null) {
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp;
            DataSource ds = (DataSource) dataSourceScanOp.getDataSource();
            if (ds.getDatasourceType() != DataSource.Type.INTERNAL_DATASET) {
                return null;
            }
            return ((DatasetDataSource) ds).getDataset();
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
            UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
            ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
            if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                FunctionIdentifier fid = f.getFunctionIdentifier();
                String dataverseName;
                String datasetName;
                if (BuiltinFunctions.EXTERNAL_LOOKUP.equals(fid)) {
                    dataverseName = AccessMethodUtils.getStringConstant(f.getArguments().get(0));
                    datasetName = AccessMethodUtils.getStringConstant(f.getArguments().get(1));
                } else if (fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                    AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                    jobGenParams.readFromFuncArgs(f.getArguments());
                    dataverseName = jobGenParams.dataverseName;
                    datasetName = jobGenParams.datasetName;
                } else {
                    throw new AlgebricksException("Unexpected function for Unnest Map: " + fid);
                }
                return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, datasetName);
            }
        }
        if (descendantOp.getInputs().isEmpty()) {
            break;
        }
        descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue();
    }
    return null;
}

Example 72 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class OptimizableOperatorSubTree method initializeDataSource.

private boolean initializeDataSource(Mutable<ILogicalOperator> subTreeOpRef) {
    AbstractLogicalOperator subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
    if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
        setDataSourceType(DataSourceType.DATASOURCE_SCAN);
        setDataSourceRef(subTreeOpRef);
        return true;
    } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
        setDataSourceType(DataSourceType.COLLECTION_SCAN);
        setDataSourceRef(subTreeOpRef);
        return true;
    } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
        // There can be multiple unnest-map or datasource-scan operators
        // if index-nested-loop-join has been applied by IntroduceJoinAccessMethodRule.
        // So, we need to traverse the whole path from the subTreeOp.
        boolean dataSourceFound = false;
        while (true) {
            if (subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
                UnnestMapOperator unnestMapOp = (UnnestMapOperator) subTreeOp;
                ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                    AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                    if (f.getFunctionIdentifier().equals(BuiltinFunctions.INDEX_SEARCH)) {
                        AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                        jobGenParams.readFromFuncArgs(f.getArguments());
                        if (jobGenParams.isPrimaryIndex()) {
                            if (getDataSourceRef() == null) {
                                setDataSourceRef(subTreeOpRef);
                                setDataSourceType(DataSourceType.PRIMARY_INDEX_LOOKUP);
                            } else {
                                // One datasource already exists. This is an additional datasource.
                                initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
                                getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.PRIMARY_INDEX_LOOKUP);
                                getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                            }
                            dataSourceFound = true;
                        }
                    } else if (f.getFunctionIdentifier().equals(BuiltinFunctions.EXTERNAL_LOOKUP)) {
                        // External lookup case
                        if (getDataSourceRef() == null) {
                            setDataSourceRef(subTreeOpRef);
                            setDataSourceType(DataSourceType.EXTERNAL_SCAN);
                        } else {
                            // One datasource already exists. This is an additional datasource.
                            initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
                            getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.EXTERNAL_SCAN);
                            getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                        }
                        dataSourceFound = true;
                    }
                }
            } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
                initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
                getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.DATASOURCE_SCAN);
                getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                dataSourceFound = true;
            } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
                initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
                getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.COLLECTION_SCAN);
                getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
            }
            // Traverse the subtree while there are operators in the path.
            if (subTreeOp.hasInputs()) {
                subTreeOpRef = subTreeOp.getInputs().get(0);
                subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
            } else {
                break;
            }
        }
        if (dataSourceFound) {
            return true;
        }
    }
    return false;
}

Example 73 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class InvertedIndexAccessMethod method analyzeGetItemFuncExpr.

public boolean analyzeGetItemFuncExpr(AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException {
    if (funcExpr.getFunctionIdentifier() != BuiltinFunctions.GET_ITEM) {
        return false;
    }
    ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
    ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
    // The second arg is the item index to be accessed. It must be a constant.
    if (arg2.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
        return false;
    }
    // If it is a variable we must track its origin in the assigns to get the original function expr.
    if (arg1.getExpressionTag() != LogicalExpressionTag.VARIABLE && arg1.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression matchedFuncExpr = null;
    // The get-item arg is function call, directly check if it's optimizable.
    if (arg1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
        matchedFuncExpr = (AbstractFunctionCallExpression) arg1;
    }
    // The get-item arg is a variable. Search the assigns and unnests for its origination function.
    int matchedAssignOrUnnestIndex = -1;
    if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
        VariableReferenceExpression varRefExpr = (VariableReferenceExpression) arg1;
        // Try to find variable ref expr in all assigns.
        for (int i = 0; i < assignsAndUnnests.size(); i++) {
            AbstractLogicalOperator op = assignsAndUnnests.get(i);
            if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
                AssignOperator assign = (AssignOperator) op;
                List<LogicalVariable> assignVars = assign.getVariables();
                List<Mutable<ILogicalExpression>> assignExprs = assign.getExpressions();
                for (int j = 0; j < assignVars.size(); j++) {
                    LogicalVariable var = assignVars.get(j);
                    if (var != varRefExpr.getVariableReference()) {
                        continue;
                    }
                    // We've matched the variable in the first assign. Now analyze the originating function.
                    ILogicalExpression matchedExpr = assignExprs.get(j).getValue();
                    if (matchedExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
                        return false;
                    }
                    matchedFuncExpr = (AbstractFunctionCallExpression) matchedExpr;
                    break;
                }
            } else {
                UnnestOperator unnest = (UnnestOperator) op;
                LogicalVariable var = unnest.getVariable();
                if (var == varRefExpr.getVariableReference()) {
                    ILogicalExpression matchedExpr = unnest.getExpressionRef().getValue();
                    if (matchedExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
                        return false;
                    }
                    AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) matchedExpr;
                    if (unnestFuncExpr.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
                        return false;
                    }
                    matchedFuncExpr = (AbstractFunctionCallExpression) unnestFuncExpr.getArguments().get(0).getValue();
                }
            }
            // We've already found a match.
            if (matchedFuncExpr != null) {
                matchedAssignOrUnnestIndex = i;
                break;
            }
        }
    }
    // Check that the matched function is optimizable by this access method.
    if (!secondLevelFuncIdents.contains(matchedFuncExpr.getFunctionIdentifier())) {
        return false;
    }
    boolean selectMatchFound = analyzeSelectSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests, matchedAssignOrUnnestIndex, analysisCtx);
    boolean joinMatchFound = analyzeJoinSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests, matchedAssignOrUnnestIndex, analysisCtx);
    if (selectMatchFound || joinMatchFound) {
        return true;
    }
    return false;
}

Example 74 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class IntroduceSelectAccessMethodRule method checkAndApplyTheSelectTransformation.

/**
     * Recursively traverse the given plan and check whether a SELECT operator exists.
     * If one is found, maintain the path from the root to SELECT operator and
     * optimize the path from the SELECT operator to the EMPTY_TUPLE_SOURCE operator
     * if it is not already optimized.
     */
protected boolean checkAndApplyTheSelectTransformation(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    boolean selectFoundAndOptimizationApplied;
    boolean isSelectOp = false;
    Mutable<ILogicalOperator> selectRefFromThisOp = null;
    SelectOperator selectOpFromThisOp = null;
    // Check the current operator pattern to see whether it is a JOIN or not.
    if (op.getOperatorTag() == LogicalOperatorTag.SELECT) {
        selectRef = opRef;
        selectOp = (SelectOperator) op;
        selectRefFromThisOp = opRef;
        selectOpFromThisOp = (SelectOperator) op;
        isSelectOp = true;
    } else {
        // This is not a SELECT operator. Remember this operator.
        afterSelectRefs.add(opRef);
    }
    // to make sure an earlier select in the path is optimized first.
    for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
        selectFoundAndOptimizationApplied = checkAndApplyTheSelectTransformation(inputOpRef, context);
        if (selectFoundAndOptimizationApplied) {
            return true;
        }
    }
    // Traverse the plan until we find a SELECT operator.
    if (isSelectOp) {
        // Restore the information from this operator since it might have been be set to null
        // if there are other select operators in the earlier path.
        selectRef = selectRefFromThisOp;
        selectOp = selectOpFromThisOp;
        // Decides the plan transformation check needs to be continued.
        // This variable is needed since we can't just return false
        // in order to keep this operator in the afterSelectRefs list.
        boolean continueCheck = true;
        // Already checked this SELECT operator? If not, this operator may be optimized.
        if (context.checkIfInDontApplySet(this, selectOp)) {
            continueCheck = false;
        }
        // For each access method, contains the information about
        // whether an available index can be applicable or not.
        Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = null;
        if (continueCheck) {
            analyzedAMs = new TreeMap<>();
        }
        // the given plan is truly optimizable or not.
        if (continueCheck && !checkSelectOpConditionAndInitSubTree(context)) {
            continueCheck = false;
        }
        // Check whether the function in the SELECT operator can be truly transformed.
        if (continueCheck && !analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(selectCond, subTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment)) {
            continueCheck = false;
        }
        // This will be used to find an applicable index on the dataset.
        if (continueCheck && !subTree.setDatasetAndTypeMetadata((MetadataProvider) context.getMetadataProvider())) {
            continueCheck = false;
        }
        if (continueCheck) {
            // Map variables to the applicable indexes and find the field name and type.
            // Then find the applicable indexes for the variables used in the SELECT condition.
            fillSubTreeIndexExprs(subTree, analyzedAMs, context);
            // Prune the access methods based on the function expression and access methods.
            pruneIndexCandidates(analyzedAMs, context, typeEnvironment);
            // Choose all indexes that will be applied.
            List<Pair<IAccessMethod, Index>> chosenIndexes = chooseAllIndexes(analyzedAMs);
            if (chosenIndexes == null || chosenIndexes.isEmpty()) {
                // We can't apply any index for this SELECT operator
                context.addToDontApplySet(this, selectRef.getValue());
                return false;
            }
            // Apply plan transformation using chosen index.
            boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context);
            context.addToDontApplySet(this, selectOp);
            if (res) {
                OperatorPropertiesUtil.typeOpRec(opRef, context);
                return res;
            }
        }
        selectRef = null;
        selectOp = null;
        afterSelectRefs.add(opRef);
    }
    // Clean the path after SELECT operator by removing the current operator in the list.
    afterSelectRefs.remove(opRef);
    return false;
}

Example 75 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class PushProjectDownRule method pushNeededProjections.

// It does not try to push above another Projection.
private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef, IOptimizationContext context, ILogicalOperator initialOp) throws AlgebricksException {
    HashSet<LogicalVariable> allP = new HashSet<LogicalVariable>();
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    VariableUtilities.getSubplanLocalLiveVariables(op, allP);
    HashSet<LogicalVariable> toProject = new HashSet<LogicalVariable>();
    for (LogicalVariable v : toPush) {
        if (allP.contains(v)) {
            toProject.add(v);
        }
    }
    if (toProject.equals(allP)) {
        // projection would be redundant, since we would project everything
        // but we can try with the children
        boolean push = false;
        if (pushThroughOp(toProject, opRef, initialOp, context).first) {
            push = true;
        }
        return push;
    } else {
        return pushAllProjectionsOnTopOf(toProject, opRef, context, initialOp);
    }
}