Examples with DecomposedPredicate org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate used on opensource projects

Example 1 with DecomposedPredicate

use of org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate in project hive by apache.

the class CompactIndexHandler method generateIndexQuery.

@Override
public void generateIndexQuery(List<Index> indexes, ExprNodeDesc predicate, ParseContext pctx, HiveIndexQueryContext queryContext) {
    Index index = indexes.get(0);
    DecomposedPredicate decomposedPredicate = decomposePredicate(predicate, index, queryContext.getQueryPartitions());
    if (decomposedPredicate == null) {
        queryContext.setQueryTasks(null);
        // abort if we couldn't pull out anything from the predicate
        return;
    }
    // pass residual predicate back out for further processing
    queryContext.setResidualPredicate(decomposedPredicate.residualPredicate);
    // setup TableScanOperator to change input format for original query
    queryContext.setIndexInputFormat(HiveCompactIndexInputFormat.class.getName());
    // Build reentrant QL for index query
    StringBuilder qlCommand = new StringBuilder("INSERT OVERWRITE DIRECTORY ");
    String tmpFile = pctx.getContext().getMRTmpPath().toUri().toString();
    queryContext.setIndexIntermediateFile(tmpFile);
    // QL includes " around file name
    qlCommand.append("\"" + tmpFile + "\" ");
    qlCommand.append("SELECT `_bucketname` ,  `_offsets` FROM ");
    qlCommand.append(HiveUtils.unparseIdentifier(index.getIndexTableName()));
    qlCommand.append(" WHERE ");
    String predicateString = decomposedPredicate.pushedPredicate.getExprString();
    qlCommand.append(predicateString);
    // generate tasks from index query string
    LOG.info("Generating tasks for re-entrant QL query: " + qlCommand.toString());
    HiveConf queryConf = new HiveConf(pctx.getConf(), CompactIndexHandler.class);
    HiveConf.setBoolVar(queryConf, HiveConf.ConfVars.COMPRESSRESULT, false);
    Driver driver = new Driver(queryConf);
    driver.compile(qlCommand.toString(), false);
    if (pctx.getConf().getBoolVar(ConfVars.HIVE_INDEX_COMPACT_BINARY_SEARCH) && useSorted) {
        // For now, only works if the predicate is a single condition
        MapWork work = null;
        String originalInputFormat = null;
        for (Task task : driver.getPlan().getRootTasks()) {
            // Otherwise something is wrong, log the problem and continue using the default format
            if (task.getWork() instanceof MapredWork) {
                if (work != null) {
                    LOG.error("Tried to use a binary search on a compact index but there were an " + "unexpected number (>1) of root level map reduce tasks in the " + "reentrant query plan.");
                    work.setInputformat(null);
                    work.setInputFormatSorted(false);
                    break;
                }
                if (task.getWork() != null) {
                    work = ((MapredWork) task.getWork()).getMapWork();
                }
                String inputFormat = work.getInputformat();
                originalInputFormat = inputFormat;
                if (inputFormat == null) {
                    inputFormat = HiveConf.getVar(pctx.getConf(), HiveConf.ConfVars.HIVEINPUTFORMAT);
                }
                // and BucketizedHiveInputFormat
                try {
                    if (!HiveInputFormat.class.isAssignableFrom(JavaUtils.loadClass(inputFormat))) {
                        work = null;
                        break;
                    }
                } catch (ClassNotFoundException e) {
                    LOG.error("Map reduce work's input format class: " + inputFormat + " was not found. " + "Cannot use the fact the compact index is sorted.");
                    work = null;
                    break;
                }
                work.setInputFormatSorted(true);
            }
        }
        if (work != null) {
            // Find the filter operator and expr node which act on the index column and mark them
            if (!findIndexColumnFilter(work.getAliasToWork().values())) {
                LOG.error("Could not locate the index column's filter operator and expr node. Cannot " + "use the fact the compact index is sorted.");
                work.setInputformat(originalInputFormat);
                work.setInputFormatSorted(false);
            }
        }
    }
    queryContext.addAdditionalSemanticInputs(driver.getPlan().getInputs());
    queryContext.setQueryTasks(driver.getPlan().getRootTasks());
    return;
}

Example 2 with DecomposedPredicate

use of org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate in project phoenix by apache.

the class PhoenixPredicateDecomposer method decomposePredicate.

public DecomposedPredicate decomposePredicate(ExprNodeDesc predicate) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("predicate - " + predicate.toString());
    }
    IndexPredicateAnalyzer analyzer = PredicateAnalyzerFactory.createPredicateAnalyzer(columnNameList, getFieldValidator());
    DecomposedPredicate decomposed = new DecomposedPredicate();
    List<IndexSearchCondition> conditions = new ArrayList<IndexSearchCondition>();
    decomposed.residualPredicate = (ExprNodeGenericFuncDesc) analyzer.analyzePredicate(predicate, conditions);
    if (!conditions.isEmpty()) {
        decomposed.pushedPredicate = analyzer.translateSearchConditions(conditions);
        try {
            searchConditionList = conditions;
            calledPPD = true;
        } catch (Exception e) {
            LOG.warn("Failed to decompose predicates", e);
            return null;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("decomposed predicate - residualPredicate: " + decomposed.residualPredicate + ", pushedPredicate: " + decomposed.pushedPredicate);
    }
    return decomposed;
}

Example 3 with DecomposedPredicate

use of org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate in project hive by apache.

the class AbstractHBaseKeyPredicateDecomposer method decomposePredicate.

public DecomposedPredicate decomposePredicate(String keyColName, ExprNodeDesc predicate) {
    IndexPredicateAnalyzer analyzer = IndexPredicateAnalyzer.createAnalyzer(true);
    analyzer.allowColumnName(keyColName);
    analyzer.setAcceptsFields(true);
    analyzer.setFieldValidator(getFieldValidator());
    DecomposedPredicate decomposed = new DecomposedPredicate();
    List<IndexSearchCondition> conditions = new ArrayList<IndexSearchCondition>();
    decomposed.residualPredicate = (ExprNodeGenericFuncDesc) analyzer.analyzePredicate(predicate, conditions);
    if (!conditions.isEmpty()) {
        decomposed.pushedPredicate = analyzer.translateSearchConditions(conditions);
        try {
            decomposed.pushedPredicateObject = getScanRange(conditions);
        } catch (Exception e) {
            LOG.warn("Failed to decompose predicates", e);
            return null;
        }
    }
    return decomposed;
}

Example 4 with DecomposedPredicate

use of org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate in project hive by apache.

the class KuduPredicateHandler method decompose.

/**
 * Analyzes the predicates and return the portion of it which
 * cannot be evaluated by Kudu during table access.
 *
 * @param predicateExpr predicate to be decomposed
 * @param schema the schema of the Kudu table
 * @return decomposed form of predicate, or null if no pushdown is possible at all
 */
public static DecomposedPredicate decompose(ExprNodeDesc predicateExpr, Schema schema) {
    IndexPredicateAnalyzer analyzer = newAnalyzer(schema);
    List<IndexSearchCondition> sConditions = new ArrayList<>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicateExpr, sConditions);
    // Nothing to decompose.
    if (sConditions.size() == 0) {
        return null;
    }
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(sConditions);
    decomposedPredicate.residualPredicate = (ExprNodeGenericFuncDesc) residualPredicate;
    return decomposedPredicate;
}

Example 5 with DecomposedPredicate

use of org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate in project hive by apache.

the class CompactIndexHandler method decomposePredicate.

/**
   * Split the predicate into the piece we can deal with (pushed), and the one we can't (residual)
   * @param predicate
   * @param index
   * @return
   */
private DecomposedPredicate decomposePredicate(ExprNodeDesc predicate, Index index, Set<Partition> queryPartitions) {
    IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(index, queryPartitions);
    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
    // split predicate into pushed (what we can handle), and residual (what we can't handle)
    ExprNodeGenericFuncDesc residualPredicate = (ExprNodeGenericFuncDesc) analyzer.analyzePredicate(predicate, searchConditions);
    if (searchConditions.size() == 0) {
        return null;
    }
    int numIndexCols = 0;
    for (IndexSearchCondition searchCondition : searchConditions) {
        if (!partitionCols.contains(searchCondition.getColumnDesc().getColumn())) {
            numIndexCols++;
        }
    }
    // For now, only works if the predicate has a single condition on an index column
    if (numIndexCols == 1) {
        useSorted = true;
    } else {
        useSorted = false;
    }
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
}