Examples with Index org.apache.asterix.metadata.entities.Index used on opensource projects

Example 6 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class MetadataTransactionContext method dropIndex.

public void dropIndex(String dataverseName, String datasetName, String indexName) {
    Index index = new Index(dataverseName, datasetName, indexName, null, null, null, null, false, false, MetadataUtil.PENDING_NO_OP);
    droppedCache.addIndexIfNotExists(index);
    logAndApply(new MetadataLogicalOperation(index, false));
}

Example 7 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IntroduceJoinAccessMethodRule method checkAndApplyJoinTransformation.

/**
     * Recursively traverse the given plan and check whether a INNERJOIN or LEFTOUTERJOIN operator exists.
     * If one is found, maintain the path from the root to the given join operator and
     * optimize the path from the given join operator to the EMPTY_TUPLE_SOURCE operator
     * if it is not already optimized.
     */
protected boolean checkAndApplyJoinTransformation(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    boolean joinFoundAndOptimizationApplied;
    // Check the current operator pattern to see whether it is a JOIN or not.
    boolean isThisOpInnerJoin = isInnerJoin(op);
    boolean isThisOpLeftOuterJoin = isLeftOuterJoin(op);
    boolean isParentOpGroupBy = hasGroupBy;
    Mutable<ILogicalOperator> joinRefFromThisOp = null;
    AbstractBinaryJoinOperator joinOpFromThisOp = null;
    if (isThisOpInnerJoin) {
        // Set join operator.
        joinRef = opRef;
        joinOp = (InnerJoinOperator) op;
        joinRefFromThisOp = opRef;
        joinOpFromThisOp = (InnerJoinOperator) op;
    } else if (isThisOpLeftOuterJoin) {
        // Set left-outer-join op.
        // The current operator is GROUP and the child of this op is LEFTOUERJOIN.
        joinRef = op.getInputs().get(0);
        joinOp = (LeftOuterJoinOperator) joinRef.getValue();
        joinRefFromThisOp = op.getInputs().get(0);
        joinOpFromThisOp = (LeftOuterJoinOperator) joinRefFromThisOp.getValue();
    }
    // to make sure an earlier join in the path is optimized first.
    for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
        joinFoundAndOptimizationApplied = checkAndApplyJoinTransformation(inputOpRef, context);
        if (joinFoundAndOptimizationApplied) {
            return true;
        }
    }
    // For a JOIN case, try to transform the given plan.
    if (isThisOpInnerJoin || isThisOpLeftOuterJoin) {
        // Restore the information from this operator since it might have been be set to null
        // if there are other join operators in the earlier path.
        joinRef = joinRefFromThisOp;
        joinOp = joinOpFromThisOp;
        boolean continueCheck = true;
        // Already checked? If not, this operator may be optimized.
        if (context.checkIfInDontApplySet(this, joinOp)) {
            continueCheck = false;
        }
        // For each access method, this contains the information about
        // whether an available index can be applicable or not.
        Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = null;
        if (continueCheck) {
            analyzedAMs = new HashMap<>();
        }
        // whether the given plan is truly optimizable or not.
        if (continueCheck && !checkJoinOpConditionAndInitSubTree(context)) {
            continueCheck = false;
        }
        // Analyze the condition of SELECT operator and initialize analyzedAMs.
        // Check whether the function in the SELECT operator can be truly transformed.
        boolean matchInLeftSubTree = false;
        boolean matchInRightSubTree = false;
        if (continueCheck) {
            if (leftSubTree.hasDataSource()) {
                matchInLeftSubTree = analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(joinCond, leftSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment);
            }
            if (rightSubTree.hasDataSource()) {
                matchInRightSubTree = analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(joinCond, rightSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment);
            }
        }
        // Find the dataset from the data-source and the record type of the dataset from the metadata.
        // This will be used to find an applicable index on the dataset.
        boolean checkLeftSubTreeMetadata = false;
        boolean checkRightSubTreeMetadata = false;
        if (continueCheck && (matchInLeftSubTree || matchInRightSubTree)) {
            // Set dataset and type metadata.
            if (matchInLeftSubTree) {
                checkLeftSubTreeMetadata = leftSubTree.setDatasetAndTypeMetadata(metadataProvider);
            }
            if (matchInRightSubTree) {
                checkRightSubTreeMetadata = rightSubTree.setDatasetAndTypeMetadata(metadataProvider);
            }
        }
        if (continueCheck && (checkLeftSubTreeMetadata || checkRightSubTreeMetadata)) {
            // Then find the applicable indexes for the variables used in the JOIN condition.
            if (checkLeftSubTreeMetadata) {
                fillSubTreeIndexExprs(leftSubTree, analyzedAMs, context);
            }
            if (checkRightSubTreeMetadata) {
                fillSubTreeIndexExprs(rightSubTree, analyzedAMs, context);
            }
            // Prune the access methods based on the function expression and access methods.
            pruneIndexCandidates(analyzedAMs, context, typeEnvironment);
            // If the right subtree (inner branch) has indexes, one of those indexes will be used.
            // Remove the indexes from the outer branch in the optimizer's consideration list for this rule.
            pruneIndexCandidatesFromOuterBranch(analyzedAMs);
            // We are going to use indexes from the inner branch.
            // If no index is available, then we stop here.
            Pair<IAccessMethod, Index> chosenIndex = chooseBestIndex(analyzedAMs);
            if (chosenIndex == null) {
                context.addToDontApplySet(this, joinOp);
                continueCheck = false;
            }
            if (continueCheck) {
                // Apply plan transformation using chosen index.
                AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
                // in GroupByOp.
                if (isThisOpLeftOuterJoin && isParentOpGroupBy) {
                    analysisCtx.setLOJGroupbyOpRef(opRef);
                    ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils.findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue());
                    analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr);
                }
                Dataset indexDataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex.second);
                // from the right subtree. The following is just a sanity check.
                if (!rightSubTree.hasDataSourceScan() && !indexDataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
                    return false;
                }
                // Finally, try to apply plan transformation using chosen index.
                boolean res = chosenIndex.first.applyJoinPlanTransformation(joinRef, leftSubTree, rightSubTree, chosenIndex.second, analysisCtx, context, isThisOpLeftOuterJoin, isParentOpGroupBy);
                // will find them.
                if (res) {
                    return res;
                }
            }
        }
        joinRef = null;
        joinOp = null;
    }
    return false;
}

Example 8 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method findMacthedExprFieldName.

private boolean findMacthedExprFieldName(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op, Dataset dataset, ARecordType recType, List<Index> datasetIndexes, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    while (descendantOp != null) {
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
            AssignOperator assignOp = (AssignOperator) descendantOp;
            List<LogicalVariable> varList = assignOp.getVariables();
            for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
                LogicalVariable var = varList.get(varIndex);
                int funcVarIndex = optFuncExpr.findLogicalVar(var);
                if (funcVarIndex == -1) {
                    continue;
                }
                List<String> fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType).second;
                if (fieldName == null) {
                    return false;
                }
                optFuncExpr.setFieldName(funcVarIndex, fieldName);
                return true;
            }
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator scanOp = (DataSourceScanOperator) descendantOp;
            List<LogicalVariable> varList = scanOp.getVariables();
            for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
                LogicalVariable var = varList.get(varIndex);
                int funcVarIndex = optFuncExpr.findLogicalVar(var);
                if (funcVarIndex == -1) {
                    continue;
                }
                // The variable value is one of the partitioning fields.
                List<String> fieldName = dataset.getPrimaryKeys().get(varIndex);
                if (fieldName == null) {
                    return false;
                }
                optFuncExpr.setFieldName(funcVarIndex, fieldName);
                return true;
            }
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
            UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
            List<LogicalVariable> varList = unnestMapOp.getVariables();
            for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
                LogicalVariable var = varList.get(varIndex);
                int funcVarIndex = optFuncExpr.findLogicalVar(var);
                if (funcVarIndex == -1) {
                    continue;
                }
                String indexName;
                Index index = null;
                ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                    AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                    FunctionIdentifier fid = f.getFunctionIdentifier();
                    if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                        throw new IllegalStateException();
                    }
                    AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                    jobGenParams.readFromFuncArgs(f.getArguments());
                    indexName = jobGenParams.indexName;
                    for (Index idx : datasetIndexes) {
                        if (idx.getIndexName().compareTo(indexName) == 0) {
                            index = idx;
                            break;
                        }
                    }
                }
                IAType metaItemType = ((MetadataProvider) context.getMetadataProvider()).findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
                ARecordType metaRecType = (ARecordType) metaItemType;
                int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recType, metaRecType);
                List<String> fieldName;
                if (varIndex >= numSecondaryKeys) {
                    fieldName = dataset.getPrimaryKeys().get(varIndex - numSecondaryKeys);
                } else {
                    fieldName = index.getKeyFieldNames().get(varIndex);
                }
                if (fieldName == null) {
                    return false;
                }
                optFuncExpr.setFieldName(funcVarIndex, fieldName);
                return true;
            }
        }
        if (descendantOp.getInputs().isEmpty()) {
            break;
        }
        descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue();
    }
    return false;
}

Example 9 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IntroduceSelectAccessMethodRule method intersectAllSecondaryIndexes.

/**
     * Construct all applicable secondary index-based access paths in the given selection plan and
     * intersect them using INTERSECT operator to guide to the common primary index search.
     * In case where the applicable index is one, we only construct one path.
     */
private boolean intersectAllSecondaryIndexes(List<Pair<IAccessMethod, Index>> chosenIndexes, Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context) throws AlgebricksException {
    Pair<IAccessMethod, Index> chosenIndex = null;
    Optional<Pair<IAccessMethod, Index>> primaryIndex = chosenIndexes.stream().filter(pair -> pair.second.isPrimaryIndex()).findFirst();
    if (chosenIndexes.size() == 1) {
        chosenIndex = chosenIndexes.get(0);
    } else if (primaryIndex.isPresent()) {
        // one primary + secondary indexes, choose the primary index directly.
        chosenIndex = primaryIndex.get();
    }
    if (chosenIndex != null) {
        AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
        return chosenIndex.first.applySelectPlanTransformation(afterSelectRefs, selectRef, subTree, chosenIndex.second, analysisCtx, context);
    }
    // Intersect all secondary indexes, and postpone the primary index search.
    Mutable<ILogicalExpression> conditionRef = selectOp.getCondition();
    List<ILogicalOperator> subRoots = new ArrayList<>();
    for (Pair<IAccessMethod, Index> pair : chosenIndexes) {
        AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(pair.first);
        subRoots.add(pair.first.createSecondaryToPrimaryPlan(conditionRef, subTree, null, pair.second, analysisCtx, AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(), afterSelectRefs), false, subTree.getDataSourceRef().getValue().getInputs().get(0).getValue().getExecutionMode() == ExecutionMode.UNPARTITIONED, context));
    }
    // Connect each secondary index utilization plan to a common intersect operator.
    ILogicalOperator primaryUnnestOp = connectAll2ndarySearchPlanWithIntersect(subRoots, context);
    subTree.getDataSourceRef().setValue(primaryUnnestOp);
    return primaryUnnestOp != null;
}

Example 10 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class MetadataCache method addDatasetIfNotExists.

public Dataset addDatasetIfNotExists(Dataset dataset) {
    synchronized (datasets) {
        synchronized (indexes) {
            // internal dataset.
            if (dataset.getDatasetType() == DatasetType.INTERNAL) {
                Index index = IndexUtil.getPrimaryIndex(dataset);
                addIndexIfNotExistsInternal(index);
            }
            Map<String, Dataset> m = datasets.get(dataset.getDataverseName());
            if (m == null) {
                m = new HashMap<>();
                datasets.put(dataset.getDataverseName(), m);
            }
            if (!m.containsKey(dataset.getDatasetName())) {
                return m.put(dataset.getDatasetName(), dataset);
            }
            return null;
        }
    }
}