Examples with IndexPredicateAnalyzer org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer used on opensource projects

Example 6 with IndexPredicateAnalyzer

use of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer in project hive by apache.

the class AccumuloPredicateHandler method newAnalyzer.

/**
 * Build an analyzer that allows comparison opts from compareOpts map, and all columns from table
 * definition.
 */
private IndexPredicateAnalyzer newAnalyzer(Configuration conf) {
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
    analyzer.clearAllowedColumnNames();
    for (String op : cOpKeyset()) {
        analyzer.addComparisonOp(op);
    }
    String[] hiveColumnNames = conf.getStrings(serdeConstants.LIST_COLUMNS);
    for (String col : hiveColumnNames) {
        analyzer.allowColumnName(col);
    }
    return analyzer;
}

Example 7 with IndexPredicateAnalyzer

use of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer in project hive by apache.

the class AccumuloPredicateHandler method getSearchConditions.

/**
 * @param conf
 *          Configuration
 * @return list of IndexSearchConditions from the filter expression.
 */
public List<IndexSearchCondition> getSearchConditions(Configuration conf) {
    final List<IndexSearchCondition> sConditions = Lists.newArrayList();
    ExprNodeDesc filterExpr = getExpression(conf);
    if (null == filterExpr) {
        return sConditions;
    }
    IndexPredicateAnalyzer analyzer = newAnalyzer(conf);
    ExprNodeDesc residual = analyzer.analyzePredicate(filterExpr, sConditions);
    if (residual != null) {
        throw new RuntimeException("Unexpected residual predicate: " + residual.getExprString());
    }
    return sConditions;
}

Example 8 with IndexPredicateAnalyzer

use of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer in project hive by apache.

the class HBaseStorageHandler method decomposePredicate.

public static DecomposedPredicate decomposePredicate(JobConf jobConf, HBaseSerDe hBaseSerDe, ExprNodeDesc predicate) {
    ColumnMapping keyMapping = hBaseSerDe.getHBaseSerdeParam().getKeyColumnMapping();
    ColumnMapping tsMapping = hBaseSerDe.getHBaseSerdeParam().getTimestampColumnMapping();
    IndexPredicateAnalyzer analyzer = HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(keyMapping.columnName, keyMapping.isComparable(), tsMapping == null ? null : tsMapping.columnName);
    List<IndexSearchCondition> conditions = new ArrayList<IndexSearchCondition>();
    ExprNodeGenericFuncDesc pushedPredicate = null;
    ExprNodeGenericFuncDesc residualPredicate = (ExprNodeGenericFuncDesc) analyzer.analyzePredicate(predicate, conditions);
    for (List<IndexSearchCondition> searchConditions : HiveHBaseInputFormatUtil.decompose(conditions).values()) {
        int scSize = searchConditions.size();
        if (scSize < 1 || 2 < scSize) {
            // Either there was nothing which could be pushed down (size = 0),
            // there were complex predicates which we don't support yet.
            // Currently supported are one of the form:
            // 1. key < 20                        (size = 1)
            // 2. key = 20                        (size = 1)
            // 3. key < 20 and key > 10           (size = 2)
            // Add to residual
            residualPredicate = extractResidualCondition(analyzer, searchConditions, residualPredicate);
            continue;
        }
        if (scSize == 2 && (searchConditions.get(0).getComparisonOp().equals(GenericUDFOPEqual.class.getName()) || searchConditions.get(1).getComparisonOp().equals(GenericUDFOPEqual.class.getName()))) {
            // If one of the predicates is =, then any other predicate with it is illegal.
            // Add to residual
            residualPredicate = extractResidualCondition(analyzer, searchConditions, residualPredicate);
            continue;
        }
        boolean sameType = sameTypeIndexSearchConditions(searchConditions);
        if (!sameType) {
            // If type for column and constant are different, we currently do not support pushing them
            residualPredicate = extractResidualCondition(analyzer, searchConditions, residualPredicate);
            continue;
        }
        TypeInfo typeInfo = searchConditions.get(0).getColumnDesc().getTypeInfo();
        if (typeInfo.getCategory() == Category.PRIMITIVE && PrimitiveObjectInspectorUtils.getPrimitiveGrouping(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) == PrimitiveGrouping.NUMERIC_GROUP) {
            // would be returned.
            if (scSize == 2) {
                boolean lowerBound = false;
                boolean upperBound = false;
                if (searchConditions.get(0).getComparisonOp().equals(GenericUDFOPEqualOrLessThan.class.getName()) || searchConditions.get(0).getComparisonOp().equals(GenericUDFOPLessThan.class.getName())) {
                    lowerBound = true;
                } else {
                    upperBound = true;
                }
                if (searchConditions.get(1).getComparisonOp().equals(GenericUDFOPEqualOrGreaterThan.class.getName()) || searchConditions.get(1).getComparisonOp().equals(GenericUDFOPGreaterThan.class.getName())) {
                    upperBound = true;
                } else {
                    lowerBound = true;
                }
                if (!upperBound || !lowerBound) {
                    // Not valid range, add to residual
                    residualPredicate = extractResidualCondition(analyzer, searchConditions, residualPredicate);
                    continue;
                }
            } else {
                // scSize == 1
                if (!searchConditions.get(0).getComparisonOp().equals(GenericUDFOPEqual.class.getName())) {
                    // Not valid range, add to residual
                    residualPredicate = extractResidualCondition(analyzer, searchConditions, residualPredicate);
                    continue;
                }
            }
        }
        // This one can be pushed
        pushedPredicate = extractStorageHandlerCondition(analyzer, searchConditions, pushedPredicate);
    }
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = pushedPredicate;
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
}

Example 9 with IndexPredicateAnalyzer

use of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer in project hive by apache.

the class BitmapIndexHandler method decomposePredicate.

/**
   * Split the predicate into the piece we can deal with (pushed), and the one we can't (residual)
   * @param predicate
   * @param index
   * @return
   */
private Map<Index, ExprNodeDesc> decomposePredicate(ExprNodeDesc predicate, List<Index> indexes, HiveIndexQueryContext queryContext) {
    Map<Index, ExprNodeDesc> indexPredicates = new HashMap<Index, ExprNodeDesc>();
    // compute overall residual
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(indexes, queryContext.getQueryPartitions());
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);
    // pass residual predicate back out for further processing
    queryContext.setResidualPredicate(residualPredicate);
    if (searchConditions.size() == 0) {
        return null;
    }
    for (Index index : indexes) {
        ArrayList<Index> in = new ArrayList<Index>(1);
        in.add(index);
        analyzer = getIndexPredicateAnalyzer(in, queryContext.getQueryPartitions());
        searchConditions = new ArrayList<IndexSearchCondition>();
        // split predicate into pushed (what we can handle), and residual (what we can't handle)
        // pushed predicate from translateSearchConditions is stored for the current index
        // This ensures that we apply all possible predicates to each index
        analyzer.analyzePredicate(predicate, searchConditions);
        if (searchConditions.size() == 0) {
            indexPredicates.put(index, null);
        } else {
            indexPredicates.put(index, analyzer.translateSearchConditions(searchConditions));
        }
    }
    return indexPredicates;
}

Example 10 with IndexPredicateAnalyzer

use of org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer in project hive by apache.

the class CompactIndexHandler method decomposePredicate.

/**
   * Split the predicate into the piece we can deal with (pushed), and the one we can't (residual)
   * @param predicate
   * @param index
   * @return
   */
private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index, Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeGenericFuncDesc residualPredicate = (ExprNodeGenericFuncDesc) analyzer.analyzePredicate(predicate, searchConditions);
    if (searchConditions.size() == 0) {
        return null;
    }
    int numIndexCols = 0;
    for (IndexSearchCondition searchCondition : searchConditions) {
        if (!partitionCols.contains(searchCondition.getColumnDesc().getColumn())) {
            numIndexCols++;
        }
    }
    // For now, only works if the predicate has a single condition on an index column
    if (numIndexCols == 1) {
        useSorted = true;
    } else {
        useSorted = false;
    }
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
}