Examples with AlgebricksException org.apache.hyracks.algebricks.common.exceptions.AlgebricksException used on opensource projects

Example 31 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class AccessMethodUtils method findLOJIsMissingFuncInGroupBy.

public static ScalarFunctionCallExpression findLOJIsMissingFuncInGroupBy(GroupByOperator lojGroupbyOp) throws AlgebricksException {
    //find IS_NULL function of which argument has the nullPlaceholder variable in the nested plan of groupby.
    ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0);
    Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
    AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue();
    boolean foundSelectNonNull = false;
    ScalarFunctionCallExpression isNullFuncExpr = null;
    AbstractLogicalOperator inputOp = subPlanRootOp;
    while (inputOp != null) {
        if (inputOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
            SelectOperator selectOp = (SelectOperator) inputOp;
            if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()).getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
                    ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp.getCondition().getValue();
                    if (notFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                        if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue()).getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_MISSING)) {
                            isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue();
                            if (isNullFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                                foundSelectNonNull = true;
                                break;
                            }
                        }
                    }
                }
            }
        }
        inputOp = inputOp.getInputs().size() > 0 ? (AbstractLogicalOperator) inputOp.getInputs().get(0).getValue() : null;
    }
    if (!foundSelectNonNull) {
        throw new AlgebricksException("Could not find the non-null select operator in GroupByOperator for LEFTOUTERJOIN plan optimization.");
    }
    return isNullFuncExpr;
}

Example 32 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class SimilarityCheckRule method getSimilarityCheckExpr.

private ScalarFunctionCallExpression getSimilarityCheckExpr(FunctionIdentifier normFuncIdent, AsterixConstantValue constVal, AbstractFunctionCallExpression funcExpr) throws AlgebricksException {
    // Remember args from original similarity function to add them to the similarity-check function later.
    ArrayList<Mutable<ILogicalExpression>> similarityArgs = null;
    ScalarFunctionCallExpression simCheckFuncExpr = null;
    // Look for jaccard function call, and GE or GT.
    if (funcExpr.getFunctionIdentifier() == BuiltinFunctions.SIMILARITY_JACCARD) {
        IAObject jaccThresh;
        if (normFuncIdent == AlgebricksBuiltinFunctions.GE) {
            if (constVal.getObject() instanceof AFloat) {
                jaccThresh = constVal.getObject();
            } else {
                jaccThresh = new AFloat((float) ((ADouble) constVal.getObject()).getDoubleValue());
            }
        } else if (normFuncIdent == AlgebricksBuiltinFunctions.GT) {
            float threshVal = 0.0f;
            if (constVal.getObject() instanceof AFloat) {
                threshVal = ((AFloat) constVal.getObject()).getFloatValue();
            } else {
                threshVal = (float) ((ADouble) constVal.getObject()).getDoubleValue();
            }
            float f = threshVal + Float.MIN_VALUE;
            if (f > 1.0f)
                f = 1.0f;
            jaccThresh = new AFloat(f);
        } else {
            return null;
        }
        similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
        similarityArgs.addAll(funcExpr.getArguments());
        similarityArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(jaccThresh))));
        simCheckFuncExpr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.SIMILARITY_JACCARD_CHECK), similarityArgs);
    }
    // Look for edit-distance function call, and LE or LT.
    if (funcExpr.getFunctionIdentifier() == BuiltinFunctions.EDIT_DISTANCE) {
        AInt32 aInt = new AInt32(0);
        try {
            aInt = (AInt32) ATypeHierarchy.convertNumericTypeObject(constVal.getObject(), ATypeTag.INTEGER);
        } catch (HyracksDataException e) {
            throw new AlgebricksException(e);
        }
        AInt32 edThresh;
        if (normFuncIdent == AlgebricksBuiltinFunctions.LE) {
            edThresh = aInt;
        } else if (normFuncIdent == AlgebricksBuiltinFunctions.LT) {
            int ed = aInt.getIntegerValue() - 1;
            if (ed < 0)
                ed = 0;
            edThresh = new AInt32(ed);
        } else {
            return null;
        }
        similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
        similarityArgs.addAll(funcExpr.getArguments());
        similarityArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(edThresh))));
        simCheckFuncExpr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.EDIT_DISTANCE_CHECK), similarityArgs);
    }
    // Preserve all annotations.
    if (simCheckFuncExpr != null) {
        simCheckFuncExpr.getAnnotations().putAll(funcExpr.getAnnotations());
    }
    return simCheckFuncExpr;
}

Example 33 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class UnnestToDataScanRule method createFeedDataSource.

private FeedDataSource createFeedDataSource(DataSourceId aqlId, String targetDataset, String sourceFeedName, String subscriptionLocation, MetadataProvider metadataProvider, FeedPolicyEntity feedPolicy, String outputType, String locations, LogicalVariable recordVar, IOptimizationContext context, List<LogicalVariable> pkVars) throws AlgebricksException {
    if (!aqlId.getDataverseName().equals(metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse().getDataverseName())) {
        return null;
    }
    Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset);
    ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType);
    Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName);
    FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset);
    ARecordType metaType = null;
    // Does dataset have meta?
    if (dataset.hasMetaPart()) {
        String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getAdapterConfiguration());
        if (metaTypeName == null) {
            throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
        }
        String dataverseName = aqlId.getDataverseName();
        if (metaTypeName.contains(".")) {
            dataverseName = metaTypeName.substring(0, metaTypeName.indexOf('.'));
            metaTypeName = metaTypeName.substring(metaTypeName.indexOf('.') + 1);
        }
        metaType = (ARecordType) metadataProvider.findType(dataverseName, metaTypeName);
    }
    // Is a change feed?
    List<IAType> pkTypes = null;
    List<List<String>> partitioningKeys = null;
    List<Integer> keySourceIndicator = null;
    List<Mutable<ILogicalExpression>> keyAccessExpression = null;
    List<ScalarFunctionCallExpression> keyAccessScalarFunctionCallExpression;
    if (ExternalDataUtils.isChangeFeed(sourceFeed.getAdapterConfiguration())) {
        keyAccessExpression = new ArrayList<>();
        keyAccessScalarFunctionCallExpression = new ArrayList<>();
        pkTypes = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPrimaryKeyType();
        partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey();
        if (dataset.hasMetaPart()) {
            keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
        }
        for (int i = 0; i < partitioningKeys.size(); i++) {
            List<String> key = partitioningKeys.get(i);
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                PlanTranslationUtil.prepareVarAndExpression(key, recordVar, pkVars, keyAccessExpression, null, context);
            } else {
                PlanTranslationUtil.prepareMetaKeyAccessExpression(key, recordVar, keyAccessExpression, pkVars, null, context);
            }
        }
        keyAccessExpression.forEach(expr -> keyAccessScalarFunctionCallExpression.add((ScalarFunctionCallExpression) expr.getValue()));
    } else {
        keyAccessExpression = null;
        keyAccessScalarFunctionCallExpression = null;
    }
    FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), feedConnection);
    feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
    return feedDataSource;
}

Example 34 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class UnnestToDataScanRule method handleFunction.

protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException {
    FunctionIdentifier fid = f.getFunctionIdentifier();
    if (fid.equals(BuiltinFunctions.DATASET)) {
        if (unnest.getPositionalVariable() != null) {
            // TODO remove this after enabling the support of positional variables in data scan
            throw new AlgebricksException("No positional variables are allowed over datasets.");
        }
        ILogicalExpression expr = f.getArguments().get(0).getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
            return false;
        }
        ConstantExpression ce = (ConstantExpression) expr;
        IAlgebricksConstantValue acv = ce.getValue();
        if (!(acv instanceof AsterixConstantValue)) {
            return false;
        }
        AsterixConstantValue acv2 = (AsterixConstantValue) acv;
        if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
            return false;
        }
        String datasetArg = ((AString) acv2.getObject()).getStringValue();
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
        String dataverseName = datasetReference.first;
        String datasetName = datasetReference.second;
        Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
        if (dataset == null) {
            throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
        }
        DataSourceId asid = new DataSourceId(dataverseName, datasetName);
        List<LogicalVariable> variables = new ArrayList<>();
        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
            int numPrimaryKeys = dataset.getPrimaryKeys().size();
            for (int i = 0; i < numPrimaryKeys; i++) {
                variables.add(context.newVar());
            }
        }
        variables.add(unnest.getVariable());
        DataSource dataSource = metadataProvider.findDataSource(asid);
        boolean hasMeta = dataSource.hasMeta();
        if (hasMeta) {
            variables.add(context.newVar());
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        addPrimaryKey(variables, dataSource, context);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        IAType[] schemaTypes = dataSource.getSchemaTypes();
        ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]);
        ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context);
        return true;
    } else if (fid.equals(BuiltinFunctions.FEED_COLLECT)) {
        if (unnest.getPositionalVariable() != null) {
            throw new AlgebricksException("No positional variables are allowed over feeds.");
        }
        String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
        String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
        String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
        String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
        String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
        String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        DataSourceId asid = new DataSourceId(dataverse, getTargetFeed);
        String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
        FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
        if (policy == null) {
            policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
            if (policy == null) {
                throw new AlgebricksException("Unknown feed policy:" + policyName);
            }
        }
        ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>();
        String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
        List<LogicalVariable> pkVars = new ArrayList<>();
        FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
        // The order for feeds is <Record-Meta-PK>
        feedDataScanOutputVariables.add(unnest.getVariable());
        // Does it produce meta?
        if (ds.hasMeta()) {
            feedDataScanOutputVariables.add(context.newVar());
        }
        // Does it produce pk?
        if (ds.isChange()) {
            feedDataScanOutputVariables.addAll(pkVars);
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        return true;
    }
    return false;
}

Example 35 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class AbstractIntroduceAccessMethodRule method pruneIndexCandidates.

/**
     * Removes irrelevant access methods candidates, based on whether the
     * expressions in the query match those in the index. For example, some
     * index may require all its expressions to be matched, and some indexes may
     * only require a match on a prefix of fields to be applicable. This methods
     * removes all index candidates indexExprs that are definitely not
     * applicable according to the expressions involved.
     *
     * @throws AlgebricksException
     */
public void pruneIndexCandidates(IAccessMethod accessMethod, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException {
    Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexExprAndVarIt = analysisCtx.getIteratorForIndexExprsAndVars();
    // Used to keep track of matched expressions (added for prefix search)
    int numMatchedKeys = 0;
    ArrayList<Integer> matchedExpressions = new ArrayList<>();
    while (indexExprAndVarIt.hasNext()) {
        Map.Entry<Index, List<Pair<Integer, Integer>>> indexExprAndVarEntry = indexExprAndVarIt.next();
        Index index = indexExprAndVarEntry.getKey();
        boolean allUsed = true;
        int lastFieldMatched = -1;
        boolean foundKeyField = false;
        matchedExpressions.clear();
        numMatchedKeys = 0;
        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
            List<String> keyField = index.getKeyFieldNames().get(i);
            final IAType keyType = index.getKeyFieldTypes().get(i);
            Iterator<Pair<Integer, Integer>> exprsAndVarIter = indexExprAndVarEntry.getValue().iterator();
            while (exprsAndVarIter.hasNext()) {
                final Pair<Integer, Integer> exprAndVarIdx = exprsAndVarIter.next();
                final IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprAndVarIdx.first);
                // expr and continue.
                if (!accessMethod.exprIsOptimizable(index, optFuncExpr)) {
                    exprsAndVarIter.remove();
                    continue;
                }
                boolean typeMatch = true;
                //Prune indexes based on field types
                List<IAType> matchedTypes = new ArrayList<>();
                //retrieve types of expressions joined/selected with an indexed field
                for (int j = 0; j < optFuncExpr.getNumLogicalVars(); j++) {
                    if (j != exprAndVarIdx.second) {
                        matchedTypes.add(optFuncExpr.getFieldType(j));
                    }
                }
                if (matchedTypes.size() < 2 && optFuncExpr.getNumLogicalVars() == 1) {
                    matchedTypes.add((IAType) ExpressionTypeComputer.INSTANCE.getType(optFuncExpr.getConstantExpr(0), context.getMetadataProvider(), typeEnvironment));
                }
                //infer type of logicalExpr based on index keyType
                matchedTypes.add((IAType) ExpressionTypeComputer.INSTANCE.getType(optFuncExpr.getLogicalExpr(exprAndVarIdx.second), null, new IVariableTypeEnvironment() {

                    @Override
                    public Object getVarType(LogicalVariable var) throws AlgebricksException {
                        if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second))) {
                            return keyType;
                        }
                        throw new IllegalArgumentException();
                    }

                    @Override
                    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables, List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
                        if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second))) {
                            return keyType;
                        }
                        throw new IllegalArgumentException();
                    }

                    @Override
                    public void setVarType(LogicalVariable var, Object type) {
                        throw new IllegalArgumentException();
                    }

                    @Override
                    public Object getType(ILogicalExpression expr) throws AlgebricksException {
                        return ExpressionTypeComputer.INSTANCE.getType(expr, null, this);
                    }

                    @Override
                    public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException {
                        throw new IllegalArgumentException();
                    }
                }));
                //for the case when jaccard similarity is measured between ordered & unordered lists
                boolean jaccardSimilarity = optFuncExpr.getFuncExpr().getFunctionIdentifier().getName().startsWith("similarity-jaccard-check");
                // Full-text search consideration: an (un)ordered list of string type can be compatible with string
                // type. i.e. an (un)ordered list can be provided as arguments to a string type field index.
                List<IAType> elementTypes = matchedTypes;
                if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS || optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION) {
                    for (int j = 0; j < matchedTypes.size(); j++) {
                        if (matchedTypes.get(j).getTypeTag() == ATypeTag.ARRAY || matchedTypes.get(j).getTypeTag() == ATypeTag.MULTISET) {
                            elementTypes.set(j, ((AbstractCollectionType) matchedTypes.get(j)).getItemType());
                        }
                    }
                }
                for (int j = 0; j < matchedTypes.size(); j++) {
                    for (int k = j + 1; k < matchedTypes.size(); k++) {
                        typeMatch &= isMatched(elementTypes.get(j), elementTypes.get(k), jaccardSimilarity);
                    }
                }
                // Check if any field name in the optFuncExpr matches.
                if (optFuncExpr.findFieldName(keyField) != -1) {
                    foundKeyField = typeMatch && optFuncExpr.getOperatorSubTree(exprAndVarIdx.second).hasDataSourceScan();
                    if (foundKeyField) {
                        matchedExpressions.add(exprAndVarIdx.first);
                        numMatchedKeys++;
                        if (lastFieldMatched == i - 1) {
                            lastFieldMatched = i;
                        }
                        break;
                    }
                }
            }
            if (!foundKeyField) {
                allUsed = false;
                // if any expression was matched, remove the non-matched expressions, otherwise the index is unusable
                if (lastFieldMatched >= 0) {
                    exprsAndVarIter = indexExprAndVarEntry.getValue().iterator();
                    while (exprsAndVarIter.hasNext()) {
                        if (!matchedExpressions.contains(exprsAndVarIter.next().first)) {
                            exprsAndVarIter.remove();
                        }
                    }
                }
                break;
            }
        }
        // are not, remove this candidate.
        if (!allUsed && accessMethod.matchAllIndexExprs()) {
            indexExprAndVarIt.remove();
            continue;
        }
        // A prefix of the index exprs may have been matched.
        if (accessMethod.matchPrefixIndexExprs()) {
            if (lastFieldMatched < 0) {
                indexExprAndVarIt.remove();
                continue;
            }
        }
        analysisCtx.putNumberOfMatchedKeys(index, Integer.valueOf(numMatchedKeys));
    }
}