Examples with IAType org.apache.asterix.om.types.IAType used on opensource projects

Example 16 with IAType

use of org.apache.asterix.om.types.IAType in project asterixdb by apache.

the class IntroduceSecondaryIndexInsertDeleteRule method injectFieldAccessesForIndexes.

private ILogicalOperator injectFieldAccessesForIndexes(IOptimizationContext context, Dataset dataset, List<Index> indexes, Map<IndexFieldId, LogicalVariable> fieldAccessVars, ARecordType recType, ARecordType metaType, LogicalVariable recordVar, LogicalVariable metaVar, ILogicalOperator currentTop, boolean afterOp) throws AlgebricksException {
    List<LogicalVariable> vars = new ArrayList<>();
    List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
    for (Index index : indexes) {
        if (index.isPrimaryIndex()) {
            continue;
        }
        List<IAType> skTypes = index.getKeyFieldTypes();
        List<List<String>> skNames = index.getKeyFieldNames();
        List<Integer> indicators = index.getKeyFieldSourceIndicators();
        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
            IndexFieldId indexFieldId = new IndexFieldId(indicators.get(i), skNames.get(i));
            if (fieldAccessVars.containsKey(indexFieldId)) {
                // already handled in a different index
                continue;
            }
            ARecordType sourceType = dataset.hasMetaPart() ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recType : metaType : recType;
            LogicalVariable sourceVar = dataset.hasMetaPart() ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recordVar : metaVar : recordVar;
            LogicalVariable fieldVar = context.newVar();
            // create record variable ref
            Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(sourceVar));
            IAType fieldType = sourceType.getSubFieldType(indexFieldId.fieldName);
            AbstractFunctionCallExpression theFieldAccessFunc;
            if (fieldType == null) {
                // Open field. must prevent inlining to maintain the cast before the primaryOp and
                // make handling of records with incorrect value type for this field easier and cleaner
                context.addNotToBeInlinedVar(fieldVar);
                // create field access
                AbstractFunctionCallExpression fieldAccessFunc = getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName);
                // create cast
                theFieldAccessFunc = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CAST_TYPE));
                // The first argument is the field
                theFieldAccessFunc.getArguments().add(new MutableObject<ILogicalExpression>(fieldAccessFunc));
                TypeCastUtils.setRequiredAndInputTypes(theFieldAccessFunc, skTypes.get(i), BuiltinType.ANY);
            } else {
                // Get the desired field position
                int pos = indexFieldId.fieldName.size() > 1 ? -1 : sourceType.getFieldIndex(indexFieldId.fieldName.get(0));
                // Field not found --> This is either an open field or a nested field. it can't be accessed by index
                theFieldAccessFunc = (pos == -1) ? getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName) : getClosedFieldAccessFunction(varRef, pos);
            }
            vars.add(fieldVar);
            exprs.add(new MutableObject<ILogicalExpression>(theFieldAccessFunc));
            fieldAccessVars.put(indexFieldId, fieldVar);
        }
    }
    // AssignOperator assigns secondary keys to their vars
    AssignOperator castedFieldAssignOperator = new AssignOperator(vars, exprs);
    return introduceNewOp(context, currentTop, castedFieldAssignOperator, afterOp);
}

Example 17 with IAType

use of org.apache.asterix.om.types.IAType in project asterixdb by apache.

the class PushFieldAccessRule method propagateFieldAccessRec.

@SuppressWarnings("unchecked")
private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context, String finalAnnot) throws AlgebricksException {
    AssignOperator access = (AssignOperator) opRef.getValue();
    Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    // rewritten into index search.
    if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2) && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
        return false;
    }
    Object annotation = op2.getAnnotations().get(IS_MOVABLE);
    if (annotation != null && !((Boolean) annotation)) {
        return false;
    }
    if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
        return false;
    }
    if (testAndModifyRedundantOp(access, op2)) {
        propagateFieldAccessRec(opRef2, context, finalAnnot);
        return true;
    }
    List<LogicalVariable> usedInAccess = new LinkedList<>();
    VariableUtilities.getUsedVariables(access, usedInAccess);
    List<LogicalVariable> produced2 = new LinkedList<>();
    if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        VariableUtilities.getLiveVariables(op2, produced2);
    } else {
        VariableUtilities.getProducedVariables(op2, produced2);
    }
    boolean pushItDown = false;
    List<LogicalVariable> inter = new ArrayList<>(usedInAccess);
    if (inter.isEmpty()) {
        // ground value
        return false;
    }
    inter.retainAll(produced2);
    if (inter.isEmpty()) {
        pushItDown = true;
    } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        GroupByOperator g = (GroupByOperator) op2;
        List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
            ILogicalExpression e = p.second.getValue();
            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
                if (inter.contains(decorVar)) {
                    inter.remove(decorVar);
                    LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
                    varMappings.add(new Pair<>(decorVar, v1));
                }
            }
        }
        if (inter.isEmpty()) {
            boolean changed = false;
            for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
                LogicalVariable v2 = context.newVar();
                LogicalVariable oldVar = access.getVariables().get(0);
                g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
                changed = true;
                access.getVariables().set(0, v2);
                VariableUtilities.substituteVariables(access, m.first, m.second, context);
            }
            if (changed) {
                context.computeAndSetTypeEnvironmentForOperator(g);
            }
            usedInAccess.clear();
            VariableUtilities.getUsedVariables(access, usedInAccess);
            pushItDown = true;
        }
    }
    if (pushItDown) {
        if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference().getValue().getInputs().get(0);
            pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
            return true;
        }
        if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
            pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
            return true;
        } else {
            for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
                HashSet<LogicalVariable> v2 = new HashSet<>();
                VariableUtilities.getLiveVariables(inp.getValue(), v2);
                if (v2.containsAll(usedInAccess)) {
                    pushAccessDown(opRef, op2, inp, context, finalAnnot);
                    return true;
                }
            }
        }
        if (op2.hasNestedPlans()) {
            AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
            for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
                for (Mutable<ILogicalOperator> root : plan.getRoots()) {
                    HashSet<LogicalVariable> v2 = new HashSet<>();
                    VariableUtilities.getLiveVariables(root.getValue(), v2);
                    if (v2.containsAll(usedInAccess)) {
                        pushAccessDown(opRef, op2, root, context, finalAnnot);
                        return true;
                    }
                }
            }
        }
        throw new AlgebricksException("Field access " + access.getExpressions().get(0).getValue() + " does not correspond to any input of operator " + op2);
    } else {
        // fields. If yes, we can equate the two variables.
        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator scan = (DataSourceScanOperator) op2;
            int n = scan.getVariables().size();
            LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
            AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions().get(0).getValue();
            ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
            LogicalExpressionTag tag = e0.getExpressionTag();
            if (tag == LogicalExpressionTag.VARIABLE) {
                VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
                if (varRef.getVariableReference() == scanRecordVar) {
                    ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
                    if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
                        IDataSource<DataSourceId> dataSource = (IDataSource<DataSourceId>) scan.getDataSource();
                        byte dsType = ((DataSource) dataSource).getDatasourceType();
                        if (dsType == DataSource.Type.FEED || dsType == DataSource.Type.LOADABLE) {
                            return false;
                        }
                        DataSourceId asid = dataSource.getId();
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
                        if (dataset == null) {
                            throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
                        }
                        if (dataset.getDatasetType() != DatasetType.INTERNAL) {
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        ConstantExpression ce = (ConstantExpression) e1;
                        IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
                        String fldName;
                        if (obj.getType().getTypeTag() == ATypeTag.STRING) {
                            fldName = ((AString) obj).getStringValue();
                        } else {
                            int pos = ((AInt32) obj).getIntegerValue();
                            String tName = dataset.getItemTypeName();
                            IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
                            if (t.getTypeTag() != ATypeTag.OBJECT) {
                                return false;
                            }
                            ARecordType rt = (ARecordType) t;
                            if (pos >= rt.getFieldNames().length) {
                                setAsFinal(access, context, finalAnnot);
                                return false;
                            }
                            fldName = rt.getFieldNames()[pos];
                        }
                        int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName);
                        if (p < 0) {
                            // not one of the partitioning fields
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        LogicalVariable keyVar = scan.getVariables().get(p);
                        access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
                        return true;
                    }
                }
            }
        }
        setAsFinal(access, context, finalAnnot);
        return false;
    }
}

Example 18 with IAType

use of org.apache.asterix.om.types.IAType in project asterixdb by apache.

the class AccessMethodUtils method createSearchKeyExpr.

/**
     * Returns the search key expression which feeds a secondary-index search. If we are optimizing a selection query then this method returns
     * the a ConstantExpression from the first constant value in the optimizable function expression.
     * If we are optimizing a join, then this method returns the VariableReferenceExpression that should feed the secondary index probe.
     *
     * @throws AlgebricksException
     */
public static Pair<ILogicalExpression, Boolean> createSearchKeyExpr(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree) throws AlgebricksException {
    if (probeSubTree == null) {
        // We are optimizing a selection query. Search key is a constant.
        // Type Checking and type promotion is done here
        IAType fieldType = optFuncExpr.getFieldType(0);
        if (optFuncExpr.getNumConstantExpr() == 0) {
            //TODO: Right now we miss on type promotion for nonpure functions
            return new Pair<>(new VariableReferenceExpression(optFuncExpr.getLogicalVar(1)), false);
        }
        ILogicalExpression constantAtRuntimeExpression = null;
        AsterixConstantValue constantValue = null;
        ATypeTag constantValueTag = null;
        constantAtRuntimeExpression = optFuncExpr.getConstantExpr(0);
        if (constantAtRuntimeExpression.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
            constantValue = (AsterixConstantValue) ((ConstantExpression) constantAtRuntimeExpression).getValue();
        }
        constantValueTag = optFuncExpr.getConstantType(0).getTypeTag();
        // type casting applied?
        boolean typeCastingApplied = false;
        // type casting happened from real (FLOAT, DOUBLE) value -> INT value?
        boolean realTypeConvertedToIntegerType = false;
        AsterixConstantValue replacedConstantValue = null;
        // if the constant type and target type does not match, we do a type conversion
        if (constantValueTag != fieldType.getTypeTag() && constantValue != null) {
            try {
                replacedConstantValue = ATypeHierarchy.getAsterixConstantValueFromNumericTypeObject(constantValue.getObject(), fieldType.getTypeTag(), true);
            } catch (HyracksDataException e) {
                throw new AlgebricksException(e);
            }
            if (replacedConstantValue != null) {
                typeCastingApplied = true;
            }
            // In this case, we need to change the search parameter. Refer to the caller section for the detail.
            switch(constantValueTag) {
                case DOUBLE:
                case FLOAT:
                    switch(fieldType.getTypeTag()) {
                        case TINYINT:
                        case SMALLINT:
                        case INTEGER:
                        case BIGINT:
                            realTypeConvertedToIntegerType = true;
                            break;
                        default:
                            break;
                    }
                default:
                    break;
            }
        }
        if (typeCastingApplied) {
            return new Pair<>(new ConstantExpression(replacedConstantValue), realTypeConvertedToIntegerType);
        } else {
            return new Pair<>(optFuncExpr.getConstantExpr(0), false);
        }
    } else {
        // We are optimizing a join query. Determine which variable feeds the secondary index.
        if (optFuncExpr.getOperatorSubTree(0) == null || optFuncExpr.getOperatorSubTree(0) == probeSubTree) {
            return new Pair<>(new VariableReferenceExpression(optFuncExpr.getLogicalVar(0)), false);
        } else {
            return new Pair<>(new VariableReferenceExpression(optFuncExpr.getLogicalVar(1)), false);
        }
    }
}

Example 19 with IAType

use of org.apache.asterix.om.types.IAType in project asterixdb by apache.

the class AccessMethodUtils method analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx.

public static boolean analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException {
    ILogicalExpression constExpression = null;
    IAType constantExpressionType = null;
    LogicalVariable fieldVar = null;
    ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
    ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
    // One of the args must be a runtime constant, and the other arg must be a variable.
    if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
        return false;
    }
    if (arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
        // The arguments of contains() function are asymmetrical, we can only use index if it is on the first argument
        if (funcExpr.getFunctionIdentifier() == BuiltinFunctions.STRING_CONTAINS || funcExpr.getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS || funcExpr.getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION) {
            return false;
        }
        IAType expressionType = constantRuntimeResultType(arg1, context, typeEnvironment);
        if (expressionType == null) {
            //Not constant at runtime
            return false;
        }
        constantExpressionType = expressionType;
        constExpression = arg1;
        VariableReferenceExpression varExpr = (VariableReferenceExpression) arg2;
        fieldVar = varExpr.getVariableReference();
    } else if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
        IAType expressionType = constantRuntimeResultType(arg2, context, typeEnvironment);
        if (expressionType == null) {
            //Not constant at runtime
            return false;
        }
        constantExpressionType = expressionType;
        constExpression = arg2;
        // yet in the full-text search.
        if (funcExpr.getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS && arg2.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
            checkFTSearchConstantExpression(constExpression);
        }
        VariableReferenceExpression varExpr = (VariableReferenceExpression) arg1;
        fieldVar = varExpr.getVariableReference();
    } else {
        return false;
    }
    // Updates the given Analysis Context by adding a new optimizable function expression.
    constructNewOptFuncExprAndAddToAnalysisCtx(funcExpr, fieldVar, constExpression, constantExpressionType, analysisCtx);
    return true;
}

Example 20 with IAType

use of org.apache.asterix.om.types.IAType in project asterixdb by apache.

the class UnnestToDataScanRule method createFeedDataSource.

private FeedDataSource createFeedDataSource(DataSourceId aqlId, String targetDataset, String sourceFeedName, String subscriptionLocation, MetadataProvider metadataProvider, FeedPolicyEntity feedPolicy, String outputType, String locations, LogicalVariable recordVar, IOptimizationContext context, List<LogicalVariable> pkVars) throws AlgebricksException {
    if (!aqlId.getDataverseName().equals(metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse().getDataverseName())) {
        return null;
    }
    Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset);
    ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType);
    Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName);
    FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset);
    ARecordType metaType = null;
    // Does dataset have meta?
    if (dataset.hasMetaPart()) {
        String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getAdapterConfiguration());
        if (metaTypeName == null) {
            throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
        }
        String dataverseName = aqlId.getDataverseName();
        if (metaTypeName.contains(".")) {
            dataverseName = metaTypeName.substring(0, metaTypeName.indexOf('.'));
            metaTypeName = metaTypeName.substring(metaTypeName.indexOf('.') + 1);
        }
        metaType = (ARecordType) metadataProvider.findType(dataverseName, metaTypeName);
    }
    // Is a change feed?
    List<IAType> pkTypes = null;
    List<List<String>> partitioningKeys = null;
    List<Integer> keySourceIndicator = null;
    List<Mutable<ILogicalExpression>> keyAccessExpression = null;
    List<ScalarFunctionCallExpression> keyAccessScalarFunctionCallExpression;
    if (ExternalDataUtils.isChangeFeed(sourceFeed.getAdapterConfiguration())) {
        keyAccessExpression = new ArrayList<>();
        keyAccessScalarFunctionCallExpression = new ArrayList<>();
        pkTypes = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPrimaryKeyType();
        partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey();
        if (dataset.hasMetaPart()) {
            keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
        }
        for (int i = 0; i < partitioningKeys.size(); i++) {
            List<String> key = partitioningKeys.get(i);
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                PlanTranslationUtil.prepareVarAndExpression(key, recordVar, pkVars, keyAccessExpression, null, context);
            } else {
                PlanTranslationUtil.prepareMetaKeyAccessExpression(key, recordVar, keyAccessExpression, pkVars, null, context);
            }
        }
        keyAccessExpression.forEach(expr -> keyAccessScalarFunctionCallExpression.add((ScalarFunctionCallExpression) expr.getValue()));
    } else {
        keyAccessExpression = null;
        keyAccessScalarFunctionCallExpression = null;
    }
    FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), feedConnection);
    feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
    return feedDataSource;
}