Examples with Dataset org.apache.asterix.metadata.entities.Dataset used on opensource projects

Example 41 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class PushFieldAccessRule method isAccessToIndexedField.

@SuppressWarnings("unchecked")
private boolean isAccessToIndexedField(AssignOperator assign, IOptimizationContext context) throws AlgebricksException {
    AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) assign.getExpressions().get(0).getValue();
    ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
    if (e0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable var = ((VariableReferenceExpression) e0).getVariableReference();
    if (context.findPrimaryKey(var) == null) {
        // not referring to a dataset record
        return false;
    }
    AbstractLogicalOperator op = assign;
    while (op.getInputs().size() == 1 && op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    }
    if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        return false;
    }
    DataSourceScanOperator scan = (DataSourceScanOperator) op;
    LogicalVariable recVar = scan.getVariables().get(scan.getVariables().size() - 1);
    if (recVar != var) {
        return false;
    }
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    DataSourceId asid = ((IDataSource<DataSourceId>) scan.getDataSource()).getId();
    Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
    if (dataset == null) {
        throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
    }
    if (dataset.getDatasetType() != DatasetType.INTERNAL) {
        return false;
    }
    final Integer pos = ConstantExpressionUtil.getIntConstant(accessFun.getArguments().get(1).getValue());
    if (pos != null) {
        String tName = dataset.getItemTypeName();
        IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
        if (t.getTypeTag() != ATypeTag.OBJECT) {
            return false;
        }
        ARecordType rt = (ARecordType) t;
        if (pos >= rt.getFieldNames().length) {
            return false;
        }
    }
    List<Index> datasetIndexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    boolean hasSecondaryIndex = false;
    for (Index index : datasetIndexes) {
        if (index.isSecondaryIndex()) {
            hasSecondaryIndex = true;
            break;
        }
    }
    return hasSecondaryIndex;
}

Example 42 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class InvertedIndexAccessMethod method copyAndReinitProbeSubTree.

/**
     * Copies the probeSubTree (using new variables), and reinitializes the probeSubTree to it.
     * Accordingly replaces the variables in the given joinCond, and the optFuncExpr.
     * Returns a reference to the original plan root.
     */
private Mutable<ILogicalOperator> copyAndReinitProbeSubTree(OptimizableOperatorSubTree probeSubTree, ILogicalExpression joinCond, IOptimizableFuncExpr optFuncExpr, List<LogicalVariable> originalSubTreePKs, List<LogicalVariable> surrogateSubTreePKs, IOptimizationContext context) throws AlgebricksException {
    probeSubTree.getPrimaryKeyVars(null, originalSubTreePKs);
    // Create two copies of the original probe subtree.
    // The first copy, which becomes the new probe subtree, will retain the primary-key and secondary-search key variables,
    // but have all other variables replaced with new ones.
    // The second copy, which will become an input to the top-level equi-join to resolve the surrogates,
    // will have all primary-key and secondary-search keys replaced, but retains all other original variables.
    // Variable replacement map for the first copy.
    LinkedHashMap<LogicalVariable, LogicalVariable> newProbeSubTreeVarMap = new LinkedHashMap<>();
    // Variable replacement map for the second copy.
    LinkedHashMap<LogicalVariable, LogicalVariable> joinInputSubTreeVarMap = new LinkedHashMap<>();
    // Init with all live vars.
    List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
    VariableUtilities.getLiveVariables(probeSubTree.getRoot(), liveVars);
    for (LogicalVariable var : liveVars) {
        joinInputSubTreeVarMap.put(var, var);
    }
    // Fill variable replacement maps.
    for (int i = 0; i < optFuncExpr.getNumLogicalVars(); i++) {
        joinInputSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), context.newVar());
        newProbeSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), optFuncExpr.getLogicalVar(i));
    }
    for (int i = 0; i < originalSubTreePKs.size(); i++) {
        LogicalVariable newPKVar = context.newVar();
        surrogateSubTreePKs.add(newPKVar);
        joinInputSubTreeVarMap.put(originalSubTreePKs.get(i), newPKVar);
        newProbeSubTreeVarMap.put(originalSubTreePKs.get(i), originalSubTreePKs.get(i));
    }
    // Create first copy.
    LogicalOperatorDeepCopyWithNewVariablesVisitor firstDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context, newProbeSubTreeVarMap);
    ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
    inferTypes(newProbeSubTree, context);
    Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree);
    // Create second copy.
    LogicalOperatorDeepCopyWithNewVariablesVisitor secondDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context, joinInputSubTreeVarMap);
    ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
    inferTypes(joinInputSubTree, context);
    probeSubTree.getRootRef().setValue(joinInputSubTree);
    // Remember the original probe subtree reference so we can return it.
    Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.getRootRef();
    // Replace the original probe subtree with its copy.
    Dataset origDataset = probeSubTree.getDataset();
    ARecordType origRecordType = probeSubTree.getRecordType();
    probeSubTree.initFromSubTree(newProbeSubTreeRootRef);
    probeSubTree.setDataset(origDataset);
    probeSubTree.setRecordType(origRecordType);
    // Replace the variables in the join condition based on the mapping of variables
    // in the new probe subtree.
    Map<LogicalVariable, LogicalVariable> varMapping = firstDeepCopyVisitor.getInputToOutputVariableMapping();
    for (Map.Entry<LogicalVariable, LogicalVariable> varMapEntry : varMapping.entrySet()) {
        if (varMapEntry.getKey() != varMapEntry.getValue()) {
            joinCond.substituteVar(varMapEntry.getKey(), varMapEntry.getValue());
        }
    }
    return originalProbeSubTreeRootRef;
}

Example 43 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class BTreeAccessMethod method applyJoinPlanTransformation.

@Override
public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
    Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
    // Determine if the index is applicable on the left or right side
    // (if both, we arbitrarily prefer the left side).
    Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex);
    OptimizableOperatorSubTree indexSubTree;
    OptimizableOperatorSubTree probeSubTree;
    // The following is just a sanity check.
    if (rightSubTree.hasDataSourceScan() && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
        indexSubTree = rightSubTree;
        probeSubTree = leftSubTree;
    } else {
        return false;
    }
    LogicalVariable newNullPlaceHolderVar = null;
    if (isLeftOuterJoin) {
        //get a new null place holder variable that is the first field variable of the primary key
        //from the indexSubTree's datasourceScanOp
        newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
    }
    ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(conditionRef, indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context);
    if (primaryIndexUnnestOp == null) {
        return false;
    }
    if (isLeftOuterJoin && hasGroupBy) {
        //reset the null place holder variable
        AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
    }
    // If there are conditions left, add a new select operator on top.
    indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
    if (conditionRef.getValue() != null) {
        SelectOperator topSelect = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar);
        topSelect.getInputs().add(indexSubTree.getRootRef());
        topSelect.setExecutionMode(ExecutionMode.LOCAL);
        context.computeAndSetTypeEnvironmentForOperator(topSelect);
        // Replace the original join with the new subtree rooted at the select op.
        joinRef.setValue(topSelect);
    } else {
        joinRef.setValue(indexSubTree.getRootRef().getValue());
    }
    return true;
}

Example 44 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class BTreeAccessMethod method createSecondaryToPrimaryPlan.

@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
    Dataset dataset = indexSubTree.getDataset();
    ARecordType recordType = indexSubTree.getRecordType();
    ARecordType metaRecordType = indexSubTree.getMetaRecordType();
    // we made sure indexSubTree has datasource scan
    AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
    List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
    int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
    // List of function expressions that will be replaced by the secondary-index search.
    // These func exprs will be removed from the select condition at the very end of this method.
    Set<ILogicalExpression> replacedFuncExprs = new HashSet<>();
    // Info on high and low keys for the BTree search predicate.
    ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys];
    ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys];
    LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys];
    LimitType[] highKeyLimits = new LimitType[numSecondaryKeys];
    boolean[] lowKeyInclusive = new boolean[numSecondaryKeys];
    boolean[] highKeyInclusive = new boolean[numSecondaryKeys];
    ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys];
    LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys];
    /* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range
         * predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate,
         * we just bail by setting this flag.*/
    boolean couldntFigureOut = false;
    boolean doneWithExprs = false;
    boolean isEqCondition = false;
    BitSet setLowKeys = new BitSet(numSecondaryKeys);
    BitSet setHighKeys = new BitSet(numSecondaryKeys);
    // Go through the func exprs listed as optimizable by the chosen index,
    // and formulate a range predicate on the secondary-index keys.
    // checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value
    // since we have a round issues when dealing with LT(<) OR GT(>) operator.
    boolean realTypeConvertedToIntegerType;
    for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
        // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
        IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprIndex.first);
        int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
        if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) {
            // If we are optimizing a join, the matching field may be the second field name.
            keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
        }
        if (keyPos < 0) {
            throw CompilationException.create(ErrorCode.NO_INDEX_FIELD_NAME_FOR_GIVEN_FUNC_EXPR);
        }
        Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
        ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
        if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
            constantAtRuntimeExpressions[keyPos] = searchKeyExpr;
            constAtRuntimeExprVars[keyPos] = context.newVar();
            searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]);
        }
        realTypeConvertedToIntegerType = returnedSearchKeyExpr.second;
        LimitType limit = getLimitType(optFuncExpr, probeSubTree);
        //
        if (realTypeConvertedToIntegerType) {
            if (limit == LimitType.HIGH_EXCLUSIVE) {
                limit = LimitType.HIGH_INCLUSIVE;
            } else if (limit == LimitType.LOW_EXCLUSIVE) {
                limit = LimitType.LOW_INCLUSIVE;
            }
        }
        switch(limit) {
            case EQUAL:
                {
                    if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) {
                        lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit;
                        lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true;
                        lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr;
                        setLowKeys.set(keyPos);
                        setHighKeys.set(keyPos);
                        isEqCondition = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            isEqCondition = true;
                            break;
                        }
                        couldntFigureOut = true;
                    }
                    // If high and low keys are set, we exit for now.
                    if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) {
                        doneWithExprs = true;
                    }
                    break;
                }
            case HIGH_EXCLUSIVE:
                {
                    if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) {
                        highKeyLimits[keyPos] = limit;
                        highKeyExprs[keyPos] = searchKeyExpr;
                        highKeyInclusive[keyPos] = false;
                    } else {
                        // (once from analyzing each side of the join)
                        if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case HIGH_INCLUSIVE:
                {
                    if (highKeyLimits[keyPos] == null) {
                        highKeyLimits[keyPos] = limit;
                        highKeyExprs[keyPos] = searchKeyExpr;
                        highKeyInclusive[keyPos] = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case LOW_EXCLUSIVE:
                {
                    if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) {
                        lowKeyLimits[keyPos] = limit;
                        lowKeyExprs[keyPos] = searchKeyExpr;
                        lowKeyInclusive[keyPos] = false;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case LOW_INCLUSIVE:
                {
                    if (lowKeyLimits[keyPos] == null) {
                        lowKeyLimits[keyPos] = limit;
                        lowKeyExprs[keyPos] = searchKeyExpr;
                        lowKeyInclusive[keyPos] = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
        if (!couldntFigureOut) {
            // Remember to remove this funcExpr later.
            replacedFuncExprs.add(analysisCtx.getMatchedFuncExpr(exprIndex.first).getFuncExpr());
        }
        if (doneWithExprs) {
            break;
        }
    }
    if (couldntFigureOut) {
        return null;
    }
    // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals
    // closed to obtain a superset of answers and leave the original selection in place.
    boolean primaryIndexPostProccessingIsNeeded = false;
    for (int i = 1; i < numSecondaryKeys; ++i) {
        if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
            Arrays.fill(lowKeyInclusive, true);
            primaryIndexPostProccessingIsNeeded = true;
            break;
        }
    }
    for (int i = 1; i < numSecondaryKeys; ++i) {
        if (highKeyInclusive[i] != highKeyInclusive[0]) {
            Arrays.fill(highKeyInclusive, true);
            primaryIndexPostProccessingIsNeeded = true;
            break;
        }
    }
    // determine cases when prefix search could be applied
    for (int i = 1; i < lowKeyExprs.length; i++) {
        if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) {
            numSecondaryKeys--;
            primaryIndexPostProccessingIsNeeded = true;
        }
    }
    if (lowKeyLimits[0] == null) {
        lowKeyInclusive[0] = true;
    }
    if (highKeyLimits[0] == null) {
        highKeyInclusive[0] = true;
    }
    // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index
    // search.
    // List of variables for the assign.
    ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
    // List of variables and expressions for the assign.
    ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<>();
    ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>();
    int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
    int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
    BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
    jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
    jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
    jobGenParams.setIsEqCondition(isEqCondition);
    jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys);
    jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys);
    ILogicalOperator inputOp = null;
    if (!assignKeyVarList.isEmpty()) {
        // Assign operator that sets the constant secondary-index search-key fields if necessary.
        AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
        // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
        assignConstantSearchKeys.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
        assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
        inputOp = assignConstantSearchKeys;
    } else if (probeSubTree == null) {
        //nonpure case
        //Make sure that the nonpure function is unpartitioned
        ILogicalOperator checkOp = dataSourceOp.getInputs().get(0).getValue();
        while (checkOp.getExecutionMode() != ExecutionMode.UNPARTITIONED) {
            if (checkOp.getInputs().size() == 1) {
                checkOp = checkOp.getInputs().get(0).getValue();
            } else {
                return null;
            }
        }
        inputOp = dataSourceOp.getInputs().get(0).getValue();
    } else {
        // All index search keys are variables.
        inputOp = probeSubTree.getRoot();
    }
    ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull);
    // Generate the rest of the upstream plan which feeds the search results into the primary index.
    AbstractUnnestMapOperator primaryIndexUnnestOp = null;
    boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
        // External dataset
        UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, retainInput, retainNull);
        indexSubTree.getDataSourceRef().setValue(externalDataAccessOp);
        return externalDataAccessOp;
    } else if (!isPrimaryIndex) {
        primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context);
    } else {
        List<Object> primaryIndexOutputTypes = new ArrayList<>();
        AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
        List<LogicalVariable> scanVariables = dataSourceOp.getVariables();
        // If not, we create a new condition based on remaining ones.
        if (!primaryIndexPostProccessingIsNeeded) {
            List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<>();
            try {
                getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs);
            } catch (CompilationException e) {
                return null;
            }
            // Generate new condition.
            if (!remainingFuncExprs.isEmpty()) {
                ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs);
                conditionRef.setValue(pulledCond);
            } else {
                conditionRef.setValue(null);
            }
        }
        // Checks whether LEFT_OUTER_UNNESTMAP operator is required.
        boolean leftOuterUnnestMapRequired = false;
        if (retainNull && retainInput) {
            leftOuterUnnestMapRequired = true;
        } else {
            leftOuterUnnestMapRequired = false;
        }
        if (conditionRef.getValue() != null) {
            // The job gen parameters are transferred to the actual job gen
            // via the UnnestMapOperator's function arguments.
            List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<>();
            jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
            // An index search is expressed as an unnest-map over an
            // index-search function.
            IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
            UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
            primaryIndexSearchFunc.setReturnsUniqueValues(true);
            if (!leftOuterUnnestMapRequired) {
                primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput);
            } else {
                primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true);
            }
        } else {
            if (!leftOuterUnnestMapRequired) {
                primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput);
            } else {
                primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true);
            }
        }
        primaryIndexUnnestOp.getInputs().add(new MutableObject<>(inputOp));
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context);
    }
    return primaryIndexUnnestOp;
}

Example 45 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class OptimizableOperatorSubTree method setDatasetAndTypeMetadata.

/**
     * Find the dataset corresponding to the datasource scan in the metadata.
     * Also sets recordType to be the type of that dataset.
     */
public boolean setDatasetAndTypeMetadata(MetadataProvider metadataProvider) throws AlgebricksException {
    String dataverseName = null;
    String datasetName = null;
    Dataset ds = null;
    ARecordType rType = null;
    List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<>();
    List<DataSourceType> dsTypes = new ArrayList<>();
    sourceOpRefs.add(getDataSourceRef());
    dsTypes.add(getDataSourceType());
    // If there are multiple datasources in the subtree, we need to find the dataset for these.
    if (getIxJoinOuterAdditionalDataSourceRefs() != null) {
        for (int i = 0; i < getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
            sourceOpRefs.add(getIxJoinOuterAdditionalDataSourceRefs().get(i));
            dsTypes.add(getIxJoinOuterAdditionalDataSourceTypes().get(i));
        }
    }
    for (int i = 0; i < sourceOpRefs.size(); i++) {
        switch(dsTypes.get(i)) {
            case DATASOURCE_SCAN:
                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) sourceOpRefs.get(i).getValue();
                IDataSource<?> datasource = dataSourceScan.getDataSource();
                if (datasource instanceof DataSource) {
                    byte dsType = ((DataSource) datasource).getDatasourceType();
                    if (dsType != DataSource.Type.INTERNAL_DATASET && dsType != DataSource.Type.EXTERNAL_DATASET) {
                        return false;
                    }
                }
                Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo(dataSourceScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case PRIMARY_INDEX_LOOKUP:
                AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) sourceOpRefs.get(i).getValue();
                ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                jobGenParams.readFromFuncArgs(f.getArguments());
                datasetName = jobGenParams.getDatasetName();
                dataverseName = jobGenParams.getDataverseName();
                break;
            case EXTERNAL_SCAN:
                UnnestMapOperator externalScan = (UnnestMapOperator) sourceOpRefs.get(i).getValue();
                datasetInfo = AnalysisUtil.getExternalDatasetInfo(externalScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case COLLECTION_SCAN:
                if (i != 0) {
                    getIxJoinOuterAdditionalDatasets().add(null);
                    getIxJoinOuterAdditionalRecordTypes().add(null);
                }
                continue;
            case NO_DATASOURCE:
            default:
                return false;
        }
        if (dataverseName == null || datasetName == null) {
            return false;
        }
        // Find the dataset corresponding to the datasource in the metadata.
        ds = metadataProvider.findDataset(dataverseName, datasetName);
        if (ds == null) {
            throw CompilationException.create(ErrorCode.NO_METADATA_FOR_DATASET, datasetName);
        }
        // Get the record type for that dataset.
        IAType itemType = metadataProvider.findType(ds.getItemTypeDataverseName(), ds.getItemTypeName());
        if (itemType.getTypeTag() != ATypeTag.OBJECT) {
            if (i == 0) {
                return false;
            } else {
                getIxJoinOuterAdditionalDatasets().add(null);
                getIxJoinOuterAdditionalRecordTypes().add(null);
            }
        }
        rType = (ARecordType) itemType;
        // Get the meta record type for that dataset.
        IAType metaItemType = metadataProvider.findType(ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
        // First index is always the primary datasource in this subtree.
        if (i == 0) {
            setDataset(ds);
            setRecordType(rType);
            setMetaRecordType((ARecordType) metaItemType);
        } else {
            getIxJoinOuterAdditionalDatasets().add(ds);
            getIxJoinOuterAdditionalRecordTypes().add(rType);
        }
        dataverseName = null;
        datasetName = null;
        ds = null;
        rType = null;
    }
    return true;
}