Examples with Dispatcher org.apache.hadoop.hive.ql.lib.Dispatcher used on opensource projects

Example 6 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class MetadataOnlyOptimizer method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new TableScanProcessor());
    opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%.*" + FileSinkOperator.getOperatorName() + "%"), new FileSinkProcessor());
    Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 7 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class NullScanTaskDispatcher method dispatch.

@Override
public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) throws SemanticException {
    Task<? extends Serializable> task = (Task<? extends Serializable>) nd;
    // create a the context for walking operators
    ParseContext parseContext = physicalContext.getParseContext();
    WalkerCtx walkerCtx = new WalkerCtx();
    List<MapWork> mapWorks = new ArrayList<MapWork>(task.getMapWork());
    Collections.sort(mapWorks, new Comparator<MapWork>() {

        @Override
        public int compare(MapWork o1, MapWork o2) {
            return o1.getName().compareTo(o2.getName());
        }
    });
    for (MapWork mapWork : mapWorks) {
        LOG.debug("Looking at: " + mapWork.getName());
        Collection<Operator<? extends OperatorDesc>> topOperators = mapWork.getAliasToWork().values();
        if (topOperators.size() == 0) {
            LOG.debug("No top operators");
            return null;
        }
        LOG.debug("Looking for table scans where optimization is applicable");
        // The dispatcher fires the processor corresponding to the closest
        // matching rule and passes the context along
        Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
        GraphWalker ogw = new PreOrderOnceWalker(disp);
        // Create a list of topOp nodes
        ArrayList<Node> topNodes = new ArrayList<Node>();
        // Get the top Nodes for this task
        for (Operator<? extends OperatorDesc> workOperator : topOperators) {
            if (parseContext.getTopOps().values().contains(workOperator)) {
                topNodes.add(workOperator);
            }
        }
        Operator<? extends OperatorDesc> reducer = task.getReducer(mapWork);
        if (reducer != null) {
            topNodes.add(reducer);
        }
        ogw.startWalking(topNodes, null);
        LOG.debug(String.format("Found %d null table scans", walkerCtx.getMetadataOnlyTableScans().size()));
        if (walkerCtx.getMetadataOnlyTableScans().size() > 0)
            processAlias(mapWork, walkerCtx.getMetadataOnlyTableScans());
    }
    return null;
}

Example 8 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class NonBlockingOpDeDupProc method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // 1. We apply the transformation
    String SEL = SelectOperator.getOperatorName();
    String FIL = FilterOperator.getOperatorName();
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", SEL + "%" + SEL + "%"), new SelectDedup(pctx));
    opRules.put(new RuleRegExp("R2", FIL + "%" + FIL + "%"), new FilterDedup());
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 9 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class PointLookupOptimizer method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // 1. Trigger transformation
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), new FilterTransformer());
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new ForwardWalker(disp);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 10 with Dispatcher

use of org.apache.hadoop.hive.ql.lib.Dispatcher in project hive by apache.

the class StatsOptimizer method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() || pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() || pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty() || // tables is being sampled and we can not optimize.
    !pctx.getNameToSplitSample().isEmpty()) {
        return pctx;
    }
    String TS = TableScanOperator.getOperatorName() + "%";
    String GBY = GroupByOperator.getOperatorName() + "%";
    String RS = ReduceSinkOperator.getOperatorName() + "%";
    String SEL = SelectOperator.getOperatorName() + "%";
    String FS = FileSinkOperator.getOperatorName() + "%";
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", TS + SEL + GBY + RS + GBY + SEL + FS), new MetaDataProcessor(pctx));
    opRules.put(new RuleRegExp("R2", TS + SEL + GBY + RS + GBY + FS), new MetaDataProcessor(pctx));
    NodeProcessorCtx soProcCtx = new StatsOptimizerProcContext();
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, soProcCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}