Examples with AlgebricksException org.apache.hyracks.algebricks.common.exceptions.AlgebricksException used on opensource projects

Example 26 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class SetAsterixPhysicalOperatorsRule method generateMergeAggregationExpressions.

private static void generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context) throws AlgebricksException {
    if (gby.getNestedPlans().size() != 1) {
        throw new AlgebricksException("External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    ILogicalPlan p0 = gby.getNestedPlans().get(0);
    if (p0.getRoots().size() != 1) {
        throw new AlgebricksException("External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context.getMergeAggregationExpressionFactory();
    Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
    AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
    if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        throw new AlgebricksException("The merge aggregation expression generation should not process a " + r0Logical.getOperatorTag() + " operator.");
    }
    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
    List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
    List<LogicalVariable> aggProducedVars = aggOp.getVariables();
    int n = aggOp.getExpressions().size();
    List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
    for (int i = 0; i < n; i++) {
        ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(aggProducedVars.get(i), aggFuncRefs.get(i).getValue(), context);
        if (mergeExpr == null) {
            throw new AlgebricksException("The aggregation function " + aggFuncRefs.get(i).getValue() + " does not have a registered intermediate aggregation function.");
        }
        mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
    }
    aggOp.setMergeExpressions(mergeExpressionRefs);
}

Example 27 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class SetAsterixPhysicalOperatorsRule method computeDefaultPhysicalOp.

private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, IOptimizationContext context) throws AlgebricksException {
    PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
    if (op.getOperatorTag().equals(LogicalOperatorTag.GROUP)) {
        GroupByOperator gby = (GroupByOperator) op;
        if (gby.getNestedPlans().size() == 1) {
            ILogicalPlan p0 = gby.getNestedPlans().get(0);
            if (p0.getRoots().size() == 1) {
                Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
                if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(LogicalOperatorTag.AGGREGATE)) {
                    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
                    boolean serializable = true;
                    for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
                        AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) exprRef.getValue();
                        if (!BuiltinFunctions.isAggregateFunctionSerializable(expr.getFunctionIdentifier())) {
                            serializable = false;
                            break;
                        }
                    }
                    if ((gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE || gby.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE)) {
                        boolean setToExternalGby = false;
                        if (serializable) {
                            // if serializable, use external group-by
                            // now check whether the serialized version aggregation function has corresponding intermediate agg
                            boolean hasIntermediateAgg = true;
                            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context.getMergeAggregationExpressionFactory();
                            List<LogicalVariable> originalVariables = aggOp.getVariables();
                            List<Mutable<ILogicalExpression>> aggExprs = aggOp.getExpressions();
                            int aggNum = aggExprs.size();
                            for (int i = 0; i < aggNum; i++) {
                                AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs.get(i).getValue();
                                AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions.makeSerializableAggregateFunctionExpression(expr.getFunctionIdentifier(), expr.getArguments());
                                if (mergeAggregationExpressionFactory.createMergeAggregation(originalVariables.get(i), serialAggExpr, context) == null) {
                                    hasIntermediateAgg = false;
                                    break;
                                }
                            }
                            // Check whether there are multiple aggregates in the sub plan.
                            // Currently, we don't support multiple aggregates in one external group-by.
                            boolean multipleAggOpsFound = false;
                            ILogicalOperator r1Logical = aggOp;
                            while (r1Logical.hasInputs()) {
                                r1Logical = r1Logical.getInputs().get(0).getValue();
                                if (r1Logical.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                                    multipleAggOpsFound = true;
                                    break;
                                }
                            }
                            if (hasIntermediateAgg && !multipleAggOpsFound) {
                                for (int i = 0; i < aggNum; i++) {
                                    AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs.get(i).getValue();
                                    AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions.makeSerializableAggregateFunctionExpression(expr.getFunctionIdentifier(), expr.getArguments());
                                    aggOp.getExpressions().get(i).setValue(serialAggExpr);
                                }
                                ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(gby.getGroupByList(), physicalOptimizationConfig.getMaxFramesExternalGroupBy(), (long) physicalOptimizationConfig.getMaxFramesExternalGroupBy() * physicalOptimizationConfig.getFrameSize());
                                generateMergeAggregationExpressions(gby, context);
                                op.setPhysicalOperator(externalGby);
                                setToExternalGby = true;
                            }
                        }
                        if (!setToExternalGby) {
                            // if not serializable or no intermediate agg, use pre-clustered group-by
                            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
                            List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
                            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
                                ILogicalExpression expr = p.second.getValue();
                                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                                    VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                                    columnList.add(varRef.getVariableReference());
                                }
                            }
                            op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll()));
                        }
                    }
                } else if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(LogicalOperatorTag.RUNNINGAGGREGATE)) {
                    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
                    List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
                    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
                        ILogicalExpression expr = p.second.getValue();
                        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                            columnList.add(varRef.getVariableReference());
                        }
                    }
                    op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll()));
                } else {
                    throw new AlgebricksException("Unsupported nested operator within a group-by: " + ((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().name());
                }
            }
        }
    }
    if (op.getPhysicalOperator() == null) {
        switch(op.getOperatorTag()) {
            case INNERJOIN:
                {
                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
                    break;
                }
            case LEFTOUTERJOIN:
                {
                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
                    break;
                }
            case UNNEST_MAP:
            case LEFT_OUTER_UNNEST_MAP:
                {
                    ILogicalExpression unnestExpr = null;
                    unnestExpr = ((AbstractUnnestMapOperator) op).getExpressionRef().getValue();
                    if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                        AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                        FunctionIdentifier fid = f.getFunctionIdentifier();
                        if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                            throw new IllegalStateException();
                        }
                        AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                        jobGenParams.readFromFuncArgs(f.getArguments());
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        DataSourceId dataSourceId = new DataSourceId(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
                        Dataset dataset = mp.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
                        IDataSourceIndex<String, DataSourceId> dsi = mp.findDataSourceIndex(jobGenParams.getIndexName(), dataSourceId);
                        INodeDomain storageDomain = mp.findNodeDomain(dataset.getNodeGroupName());
                        if (dsi == null) {
                            throw new AlgebricksException("Could not find index " + jobGenParams.getIndexName() + " for dataset " + dataSourceId);
                        }
                        IndexType indexType = jobGenParams.getIndexType();
                        boolean requiresBroadcast = jobGenParams.getRequiresBroadcast();
                        switch(indexType) {
                            case BTREE:
                                {
                                    BTreeJobGenParams btreeJobGenParams = new BTreeJobGenParams();
                                    btreeJobGenParams.readFromFuncArgs(f.getArguments());
                                    op.setPhysicalOperator(new BTreeSearchPOperator(dsi, storageDomain, requiresBroadcast, btreeJobGenParams.isPrimaryIndex(), btreeJobGenParams.isEqCondition(), btreeJobGenParams.getLowKeyVarList(), btreeJobGenParams.getHighKeyVarList()));
                                    break;
                                }
                            case RTREE:
                                {
                                    op.setPhysicalOperator(new RTreeSearchPOperator(dsi, storageDomain, requiresBroadcast));
                                    break;
                                }
                            case SINGLE_PARTITION_WORD_INVIX:
                            case SINGLE_PARTITION_NGRAM_INVIX:
                                {
                                    op.setPhysicalOperator(new InvertedIndexPOperator(dsi, storageDomain, requiresBroadcast, false));
                                    break;
                                }
                            case LENGTH_PARTITIONED_WORD_INVIX:
                            case LENGTH_PARTITIONED_NGRAM_INVIX:
                                {
                                    op.setPhysicalOperator(new InvertedIndexPOperator(dsi, storageDomain, requiresBroadcast, true));
                                    break;
                                }
                            default:
                                {
                                    throw new NotImplementedException(indexType + " indexes are not implemented.");
                                }
                        }
                    }
                    break;
                }
        }
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan p : nested.getNestedPlans()) {
            setPhysicalOperators(p, context);
        }
    }
    for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
        computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), context);
    }
}

Example 28 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method getDataset.

private Dataset getDataset(AbstractLogicalOperator op, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    while (descendantOp != null) {
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp;
            DataSource ds = (DataSource) dataSourceScanOp.getDataSource();
            if (ds.getDatasourceType() != DataSource.Type.INTERNAL_DATASET) {
                return null;
            }
            return ((DatasetDataSource) ds).getDataset();
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
            UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
            ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
            if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                FunctionIdentifier fid = f.getFunctionIdentifier();
                String dataverseName;
                String datasetName;
                if (BuiltinFunctions.EXTERNAL_LOOKUP.equals(fid)) {
                    dataverseName = AccessMethodUtils.getStringConstant(f.getArguments().get(0));
                    datasetName = AccessMethodUtils.getStringConstant(f.getArguments().get(1));
                } else if (fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                    AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                    jobGenParams.readFromFuncArgs(f.getArguments());
                    dataverseName = jobGenParams.dataverseName;
                    datasetName = jobGenParams.datasetName;
                } else {
                    throw new AlgebricksException("Unexpected function for Unnest Map: " + fid);
                }
                return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, datasetName);
            }
        }
        if (descendantOp.getInputs().isEmpty()) {
            break;
        }
        descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue();
    }
    return null;
}

Example 29 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class InvertedIndexAccessMethod method isEditDistanceFuncSelectOptimizable.

private boolean isEditDistanceFuncSelectOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) throws AlgebricksException {
    // Check for panic in selection query.
    // TODO: Panic also depends on prePost which is currently hardcoded to be true.
    AsterixConstantValue listOrStrConstVal = (AsterixConstantValue) ((ConstantExpression) optFuncExpr.getConstantExpr(0)).getValue();
    IAObject listOrStrObj = listOrStrConstVal.getObject();
    ATypeTag typeTag = listOrStrObj.getType().getTypeTag();
    if (!isEditDistanceFuncCompatible(typeTag, index.getIndexType())) {
        return false;
    }
    AsterixConstantValue intConstVal = (AsterixConstantValue) ((ConstantExpression) optFuncExpr.getConstantExpr(1)).getValue();
    IAObject intObj = intConstVal.getObject();
    AInt32 edThresh = null;
    // Apply type casting based on numeric types of the input to INTEGER type.
    try {
        edThresh = (AInt32) ATypeHierarchy.convertNumericTypeObject(intObj, ATypeTag.INTEGER);
    } catch (HyracksDataException e) {
        throw new AlgebricksException(e);
    }
    int mergeThreshold = 0;
    if (typeTag == ATypeTag.STRING) {
        AString astr = (AString) listOrStrObj;
        // Compute merge threshold depending on the query grams contain pre- and postfixing
        if (optFuncExpr.containsPartialField()) {
            mergeThreshold = (astr.getStringValue().length() - index.getGramLength() + 1) - edThresh.getIntegerValue() * index.getGramLength();
        } else {
            mergeThreshold = (astr.getStringValue().length() + index.getGramLength() - 1) - edThresh.getIntegerValue() * index.getGramLength();
        }
    }
    if ((typeTag == ATypeTag.ARRAY) && (index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX)) {
        IACollection alist = (IACollection) listOrStrObj;
        // Compute merge threshold.
        mergeThreshold = alist.size() - edThresh.getIntegerValue();
    }
    if (mergeThreshold <= 0) {
        // We cannot use index to optimize expr.
        return false;
    }
    return true;
}

Example 30 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class IntroduceSelectAccessMethodRule method intersectAllSecondaryIndexes.

/**
     * Construct all applicable secondary index-based access paths in the given selection plan and
     * intersect them using INTERSECT operator to guide to the common primary index search.
     * In case where the applicable index is one, we only construct one path.
     */
private boolean intersectAllSecondaryIndexes(List<Pair<IAccessMethod, Index>> chosenIndexes, Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context) throws AlgebricksException {
    Pair<IAccessMethod, Index> chosenIndex = null;
    Optional<Pair<IAccessMethod, Index>> primaryIndex = chosenIndexes.stream().filter(pair -> pair.second.isPrimaryIndex()).findFirst();
    if (chosenIndexes.size() == 1) {
        chosenIndex = chosenIndexes.get(0);
    } else if (primaryIndex.isPresent()) {
        // one primary + secondary indexes, choose the primary index directly.
        chosenIndex = primaryIndex.get();
    }
    if (chosenIndex != null) {
        AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first);
        return chosenIndex.first.applySelectPlanTransformation(afterSelectRefs, selectRef, subTree, chosenIndex.second, analysisCtx, context);
    }
    // Intersect all secondary indexes, and postpone the primary index search.
    Mutable<ILogicalExpression> conditionRef = selectOp.getCondition();
    List<ILogicalOperator> subRoots = new ArrayList<>();
    for (Pair<IAccessMethod, Index> pair : chosenIndexes) {
        AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(pair.first);
        subRoots.add(pair.first.createSecondaryToPrimaryPlan(conditionRef, subTree, null, pair.second, analysisCtx, AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(), afterSelectRefs), false, subTree.getDataSourceRef().getValue().getInputs().get(0).getValue().getExecutionMode() == ExecutionMode.UNPARTITIONED, context));
    }
    // Connect each secondary index utilization plan to a common intersect operator.
    ILogicalOperator primaryUnnestOp = connectAll2ndarySearchPlanWithIntersect(subRoots, context);
    subTree.getDataSourceRef().setValue(primaryUnnestOp);
    return primaryUnnestOp != null;
}