Examples with PreOrderOnceWalker org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker used on opensource projects

Example 1 with PreOrderOnceWalker

use of org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker in project hive by apache.

the class SyntheticJoinPredicate method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    boolean enabled = false;
    String queryEngine = pctx.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE);
    if (queryEngine.equals("tez") && pctx.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING)) {
        enabled = true;
    } else if ((queryEngine.equals("spark") && pctx.getConf().getBoolVar(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING))) {
        enabled = true;
    }
    if (!enabled) {
        return pctx;
    }
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + ".*" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinSynthetic());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    SyntheticContext context = new SyntheticContext(pctx);
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context);
    GraphWalker ogw = new PreOrderOnceWalker(disp);
    // Create a list of top op nodes
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 2 with PreOrderOnceWalker

use of org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker in project hive by apache.

the class NullScanTaskDispatcher method dispatch.

@Override
public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) throws SemanticException {
    Task<? extends Serializable> task = (Task<? extends Serializable>) nd;
    // create a the context for walking operators
    ParseContext parseContext = physicalContext.getParseContext();
    WalkerCtx walkerCtx = new WalkerCtx();
    List<MapWork> mapWorks = new ArrayList<MapWork>(task.getMapWork());
    Collections.sort(mapWorks, new Comparator<MapWork>() {

        @Override
        public int compare(MapWork o1, MapWork o2) {
            return o1.getName().compareTo(o2.getName());
        }
    });
    for (MapWork mapWork : mapWorks) {
        LOG.debug("Looking at: " + mapWork.getName());
        Collection<Operator<? extends OperatorDesc>> topOperators = mapWork.getAliasToWork().values();
        if (topOperators.size() == 0) {
            LOG.debug("No top operators");
            return null;
        }
        LOG.debug("Looking for table scans where optimization is applicable");
        // The dispatcher fires the processor corresponding to the closest
        // matching rule and passes the context along
        Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
        GraphWalker ogw = new PreOrderOnceWalker(disp);
        // Create a list of topOp nodes
        ArrayList<Node> topNodes = new ArrayList<Node>();
        // Get the top Nodes for this task
        for (Operator<? extends OperatorDesc> workOperator : topOperators) {
            if (parseContext.getTopOps().values().contains(workOperator)) {
                topNodes.add(workOperator);
            }
        }
        Operator<? extends OperatorDesc> reducer = task.getReducer(mapWork);
        if (reducer != null) {
            topNodes.add(reducer);
        }
        ogw.startWalking(topNodes, null);
        LOG.debug(String.format("Found %d null table scans", walkerCtx.getMetadataOnlyTableScans().size()));
        if (walkerCtx.getMetadataOnlyTableScans().size() > 0)
            processAlias(mapWork, walkerCtx.getMetadataOnlyTableScans());
    }
    return null;
}

Example 3 with PreOrderOnceWalker

use of org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker in project hive by apache.

the class RewriteCanApplyCtx method populateRewriteVars.

/**
   * This method walks all the nodes starting from topOp TableScanOperator node
   * and invokes methods from {@link RewriteCanApplyProcFactory} for each of the rules
   * added to the opRules map. We use the {@link PreOrderOnceWalker} for a pre-order
   * traversal of the operator tree.
   *
   * The methods from {@link RewriteCanApplyProcFactory} set appropriate values in
   * {@link RewriteVars} enum.
   *
   * @param topOp
   * @throws SemanticException
   */
void populateRewriteVars(TableScanOperator topOp) throws SemanticException {
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    //^TS%[(SEL%)|(FIL%)]*GRY%[(FIL%)]*RS%[(FIL%)]*GRY%
    opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%[(" + SelectOperator.getOperatorName() + "%)|(" + FilterOperator.getOperatorName() + "%)]*" + GroupByOperator.getOperatorName() + "%[" + FilterOperator.getOperatorName() + "%]*" + ReduceSinkOperator.getOperatorName() + "%[" + FilterOperator.getOperatorName() + "%]*" + GroupByOperator.getOperatorName() + "%"), RewriteCanApplyProcFactory.canApplyOnTableScanOperator(topOp));
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this);
    GraphWalker ogw = new PreOrderOnceWalker(disp);
    // Create a list of topop nodes
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.add(topOp);
    try {
        ogw.startWalking(topNodes, null);
    } catch (SemanticException e) {
        LOG.error("Exception in walking operator tree. Rewrite variables not populated");
        LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
        throw new SemanticException(e.getMessage(), e);
    }
}