Examples with Index org.apache.asterix.metadata.entities.Index used on opensource projects

Example 31 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class PushFieldAccessRule method isAccessToIndexedField.

@SuppressWarnings("unchecked")
private boolean isAccessToIndexedField(AssignOperator assign, IOptimizationContext context) throws AlgebricksException {
    AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) assign.getExpressions().get(0).getValue();
    ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
    if (e0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable var = ((VariableReferenceExpression) e0).getVariableReference();
    if (context.findPrimaryKey(var) == null) {
        // not referring to a dataset record
        return false;
    }
    AbstractLogicalOperator op = assign;
    while (op.getInputs().size() == 1 && op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    }
    if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        return false;
    }
    DataSourceScanOperator scan = (DataSourceScanOperator) op;
    LogicalVariable recVar = scan.getVariables().get(scan.getVariables().size() - 1);
    if (recVar != var) {
        return false;
    }
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    DataSourceId asid = ((IDataSource<DataSourceId>) scan.getDataSource()).getId();
    Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
    if (dataset == null) {
        throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
    }
    if (dataset.getDatasetType() != DatasetType.INTERNAL) {
        return false;
    }
    final Integer pos = ConstantExpressionUtil.getIntConstant(accessFun.getArguments().get(1).getValue());
    if (pos != null) {
        String tName = dataset.getItemTypeName();
        IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
        if (t.getTypeTag() != ATypeTag.OBJECT) {
            return false;
        }
        ARecordType rt = (ARecordType) t;
        if (pos >= rt.getFieldNames().length) {
            return false;
        }
    }
    List<Index> datasetIndexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    boolean hasSecondaryIndex = false;
    for (Index index : datasetIndexes) {
        if (index.isSecondaryIndex()) {
            hasSecondaryIndex = true;
            break;
        }
    }
    return hasSecondaryIndex;
}

Example 32 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class AbstractIntroduceAccessMethodRule method fillAllIndexExprs.

protected void fillAllIndexExprs(OptimizableOperatorSubTree subTree, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
    int optFuncExprIndex = 0;
    List<Index> datasetIndexes = new ArrayList<>();
    LogicalVariable datasetMetaVar = null;
    LogicalVariable datasetRecordVar = null;
    if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN) {
        datasetIndexes = metadataProvider.getDatasetIndexes(subTree.getDataset().getDataverseName(), subTree.getDataset().getDatasetName());
        List<LogicalVariable> datasetVars = subTree.getDataSourceVariables();
        if (subTree.getDataset().hasMetaPart()) {
            datasetMetaVar = datasetVars.get(datasetVars.size() - 1);
            datasetRecordVar = datasetVars.get(datasetVars.size() - 2);
        } else {
            datasetRecordVar = datasetVars.get(datasetVars.size() - 1);
        }
    }
    for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) {
        // Try to match variables from optFuncExpr to assigns or unnests.
        for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.getAssignsAndUnnests().size(); assignOrUnnestIndex++) {
            AbstractLogicalOperator op = subTree.getAssignsAndUnnests().get(assignOrUnnestIndex);
            if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
                analyzeAssignOp((AssignOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetRecordVar, datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx);
            } else {
                analyzeUnnestOp((UnnestOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetRecordVar, datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx);
            }
        }
        // Try to match variables from optFuncExpr to datasourcescan if not
        // already matched in assigns.
        List<LogicalVariable> dsVarList = subTree.getDataSourceVariables();
        matchVarsFromOptFuncExprToDataSourceScan(optFuncExpr, optFuncExprIndex, datasetIndexes, dsVarList, subTree, analysisCtx, context, false);
        // If there is one more datasource in the subtree, we need to scan that datasource, too.
        List<LogicalVariable> additionalDsVarList = null;
        if (subTree.hasIxJoinOuterAdditionalDataSource()) {
            additionalDsVarList = new ArrayList<>();
            for (int i = 0; i < subTree.getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
                additionalDsVarList.addAll(subTree.getIxJoinOuterAdditionalDataSourceVariables(i));
            }
            matchVarsFromOptFuncExprToDataSourceScan(optFuncExpr, optFuncExprIndex, datasetIndexes, additionalDsVarList, subTree, analysisCtx, context, true);
        }
        optFuncExprIndex++;
    }
}

Example 33 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class AbstractIntroduceAccessMethodRule method chooseAllIndexes.

/**
     * Choose all indexes that match the given access method. These indexes will be used as index-search
     * to replace the given predicates in a SELECT operator. Also, if there are multiple same type of indexes
     * on the same field, only of them will be chosen. Allowed cases (AccessMethod, IndexType) are:
     * [BTreeAccessMethod , IndexType.BTREE], [RTreeAccessMethod , IndexType.RTREE],
     * [InvertedIndexAccessMethod, IndexType.SINGLE_PARTITION_WORD_INVIX || SINGLE_PARTITION_NGRAM_INVIX ||
     * LENGTH_PARTITIONED_WORD_INVIX || LENGTH_PARTITIONED_NGRAM_INVIX]
     */
protected List<Pair<IAccessMethod, Index>> chooseAllIndexes(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
    List<Pair<IAccessMethod, Index>> result = new ArrayList<>();
    // Use variables (fields) to the index types map to check which type of indexes are applied for the vars.
    Map<List<Pair<Integer, Integer>>, List<IndexType>> resultVarsToIndexTypesMap = new HashMap<>();
    Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
    while (amIt.hasNext()) {
        Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next();
        AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
        Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = analysisCtx.getIteratorForIndexExprsAndVars();
        while (indexIt.hasNext()) {
            Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
            IAccessMethod chosenAccessMethod = amEntry.getKey();
            Index chosenIndex = indexEntry.getKey();
            IndexType indexType = chosenIndex.getIndexType();
            boolean isKeywordOrNgramIndexChosen = indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX || indexType == IndexType.SINGLE_PARTITION_WORD_INVIX || indexType == IndexType.SINGLE_PARTITION_NGRAM_INVIX;
            if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && indexType == IndexType.BTREE) || (chosenAccessMethod == RTreeAccessMethod.INSTANCE && indexType == IndexType.RTREE) || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && isKeywordOrNgramIndexChosen)) {
                if (resultVarsToIndexTypesMap.containsKey(indexEntry.getValue())) {
                    List<IndexType> appliedIndexTypes = resultVarsToIndexTypesMap.get(indexEntry.getValue());
                    if (!appliedIndexTypes.contains(indexType)) {
                        appliedIndexTypes.add(indexType);
                        result.add(new Pair<>(chosenAccessMethod, chosenIndex));
                    }
                } else {
                    List<IndexType> addedIndexTypes = new ArrayList<>();
                    addedIndexTypes.add(indexType);
                    resultVarsToIndexTypesMap.put(indexEntry.getValue(), addedIndexTypes);
                    result.add(new Pair<>(chosenAccessMethod, chosenIndex));
                }
            }
        }
    }
    return result;
}

Example 34 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class AbstractIntroduceAccessMethodRule method fillIndexExprs.

/**
     * Finds secondary indexes whose keys include fieldName, and adds a mapping
     * in analysisCtx.indexEsprs from that index to the a corresponding
     * optimizable function expression.
     *
     * @return true if a candidate index was added to foundIndexExprs, false
     *         otherwise
     * @throws AlgebricksException
     */
protected boolean fillIndexExprs(List<Index> datasetIndexes, List<String> fieldName, IAType fieldType, IOptimizableFuncExpr optFuncExpr, int matchedFuncExprIndex, int varIdx, OptimizableOperatorSubTree matchedSubTree, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException {
    List<Index> indexCandidates = new ArrayList<>();
    // fieldName
    for (Index index : datasetIndexes) {
        // Need to also verify the index is pending no op
        if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == MetadataUtil.PENDING_NO_OP) {
            indexCandidates.add(index);
            if (optFuncExpr.getFieldType(varIdx) == BuiltinType.AMISSING || optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY) {
                optFuncExpr.setFieldType(varIdx, index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName)));
            }
            analysisCtx.addIndexExpr(matchedSubTree.getDataset(), index, matchedFuncExprIndex, varIdx);
        }
    }
    // No index candidates for fieldName.
    if (indexCandidates.isEmpty()) {
        return false;
    }
    return true;
}

Example 35 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    if (!checkIfRuleIsApplicable(opRef, context)) {
        return false;
    }
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    typeEnvironment = context.getOutputTypeEnvironment(op);
    ILogicalExpression condExpr = ((SelectOperator) op).getCondition().getValue();
    AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr, context, typeEnvironment);
    if (analysisCtx.getMatchedFuncExprs().isEmpty()) {
        return false;
    }
    Dataset dataset = getDataset(op, context);
    List<String> filterFieldName = null;
    ARecordType recType = null;
    if (dataset != null && dataset.getDatasetType() == DatasetType.INTERNAL) {
        filterFieldName = DatasetUtil.getFilterField(dataset);
        IAType itemType = ((MetadataProvider) context.getMetadataProvider()).findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
        if (itemType.getTypeTag() == ATypeTag.OBJECT) {
            recType = (ARecordType) itemType;
        }
    }
    if (filterFieldName == null || recType == null) {
        return false;
    }
    List<Index> datasetIndexes = ((MetadataProvider) context.getMetadataProvider()).getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<>();
    for (int i = 0; i < analysisCtx.getMatchedFuncExprs().size(); i++) {
        IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(i);
        boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes, context);
        if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) {
            optFuncExprs.add(optFuncExpr);
        }
    }
    if (optFuncExprs.isEmpty()) {
        return false;
    }
    changePlan(optFuncExprs, op, dataset, context);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    context.addToDontApplySet(this, op);
    return true;
}