Search in sources :

Example 1 with TopNKeyPushdownProcessor

use of org.apache.hadoop.hive.ql.optimizer.topnkey.TopNKeyPushdownProcessor in project hive by apache.

the class TezCompiler method runTopNKeyOptimization.

private static void runTopNKeyOptimization(OptimizeTezProcContext procCtx) throws SemanticException {
    if (!procCtx.conf.getBoolVar(ConfVars.HIVE_OPTIMIZE_TOPNKEY)) {
        return;
    }
    Map<SemanticRule, SemanticNodeProcessor> opRules = new LinkedHashMap<SemanticRule, SemanticNodeProcessor>();
    opRules.put(new RuleRegExp("Top n key optimization", ReduceSinkOperator.getOperatorName() + "%"), new TopNKeyProcessor(HiveConf.getIntVar(procCtx.conf, HiveConf.ConfVars.HIVE_MAX_TOPN_ALLOWED), HiveConf.getFloatVar(procCtx.conf, ConfVars.HIVE_TOPN_EFFICIENCY_THRESHOLD), HiveConf.getIntVar(procCtx.conf, ConfVars.HIVE_TOPN_EFFICIENCY_CHECK_BATCHES), HiveConf.getIntVar(procCtx.conf, ConfVars.HIVE_TOPN_MAX_NUMBER_OF_PARTITIONS)));
    opRules.put(new RuleRegExp("Top n key pushdown", TopNKeyOperator.getOperatorName() + "%"), new TopNKeyPushdownProcessor());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    SemanticDispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(procCtx.parseContext.getTopOps().values());
    SemanticGraphWalker ogw = new DefaultGraphWalker(disp);
    ogw.startWalking(topNodes, null);
}
Also used : SemanticRule(org.apache.hadoop.hive.ql.lib.SemanticRule) DefaultRuleDispatcher(org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher) Node(org.apache.hadoop.hive.ql.lib.Node) DefaultGraphWalker(org.apache.hadoop.hive.ql.lib.DefaultGraphWalker) RuleRegExp(org.apache.hadoop.hive.ql.lib.RuleRegExp) ArrayList(java.util.ArrayList) SemanticGraphWalker(org.apache.hadoop.hive.ql.lib.SemanticGraphWalker) TopNKeyProcessor(org.apache.hadoop.hive.ql.optimizer.topnkey.TopNKeyProcessor) LinkedHashMap(java.util.LinkedHashMap) TopNKeyPushdownProcessor(org.apache.hadoop.hive.ql.optimizer.topnkey.TopNKeyPushdownProcessor) SemanticDispatcher(org.apache.hadoop.hive.ql.lib.SemanticDispatcher) SemanticNodeProcessor(org.apache.hadoop.hive.ql.lib.SemanticNodeProcessor)

Aggregations

ArrayList (java.util.ArrayList)1 LinkedHashMap (java.util.LinkedHashMap)1 DefaultGraphWalker (org.apache.hadoop.hive.ql.lib.DefaultGraphWalker)1 DefaultRuleDispatcher (org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher)1 Node (org.apache.hadoop.hive.ql.lib.Node)1 RuleRegExp (org.apache.hadoop.hive.ql.lib.RuleRegExp)1 SemanticDispatcher (org.apache.hadoop.hive.ql.lib.SemanticDispatcher)1 SemanticGraphWalker (org.apache.hadoop.hive.ql.lib.SemanticGraphWalker)1 SemanticNodeProcessor (org.apache.hadoop.hive.ql.lib.SemanticNodeProcessor)1 SemanticRule (org.apache.hadoop.hive.ql.lib.SemanticRule)1 TopNKeyProcessor (org.apache.hadoop.hive.ql.optimizer.topnkey.TopNKeyProcessor)1 TopNKeyPushdownProcessor (org.apache.hadoop.hive.ql.optimizer.topnkey.TopNKeyPushdownProcessor)1