Examples with SemanticGraphWalker org.apache.hadoop.hive.ql.lib.SemanticGraphWalker used on opensource projects

Example 36 with SemanticGraphWalker

use of org.apache.hadoop.hive.ql.lib.SemanticGraphWalker in project hive by apache.

the class NullScanTaskDispatcher method dispatch.

@Override
public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) throws SemanticException {
    Task<?> task = (Task<?>) nd;
    // create a the context for walking operators
    ParseContext parseContext = physicalContext.getParseContext();
    WalkerCtx walkerCtx = new WalkerCtx();
    List<MapWork> mapWorks = new ArrayList<MapWork>(task.getMapWork());
    Collections.sort(mapWorks, new Comparator<MapWork>() {

        @Override
        public int compare(MapWork o1, MapWork o2) {
            return o1.getName().compareTo(o2.getName());
        }
    });
    for (MapWork mapWork : mapWorks) {
        LOG.debug("Looking at: {}", mapWork.getName());
        Collection<Operator<? extends OperatorDesc>> topOperators = mapWork.getAliasToWork().values();
        if (topOperators.isEmpty()) {
            LOG.debug("No top operators");
            return null;
        }
        LOG.debug("Looking for table scans where optimization is applicable");
        // The dispatcher fires the processor corresponding to the closest
        // matching rule and passes the context along
        SemanticDispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
        SemanticGraphWalker ogw = new PreOrderOnceWalker(disp);
        // Create a list of topOp nodes
        ArrayList<Node> topNodes = new ArrayList<>();
        // Get the top Nodes for this task
        Collection<TableScanOperator> topOps = parseContext.getTopOps().values();
        for (Operator<? extends OperatorDesc> workOperator : topOperators) {
            if (topOps.contains(workOperator)) {
                topNodes.add(workOperator);
            }
        }
        Operator<? extends OperatorDesc> reducer = task.getReducer(mapWork);
        if (reducer != null) {
            topNodes.add(reducer);
        }
        ogw.startWalking(topNodes, null);
        int scanTableSize = walkerCtx.getMetadataOnlyTableScans().size();
        LOG.debug("Found {} null table scans", scanTableSize);
        if (scanTableSize > 0) {
            processTableScans(mapWork, walkerCtx.getMetadataOnlyTableScans());
        }
    }
    return null;
}

Example 37 with SemanticGraphWalker

use of org.apache.hadoop.hive.ql.lib.SemanticGraphWalker in project hive by apache.

the class AnnotateWithOpTraits method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    AnnotateOpTraitsProcCtx annotateCtx = new AnnotateOpTraitsProcCtx(pctx);
    // create a walker which walks the tree in a BFS manner while maintaining the
    // operator stack. The dispatcher generates the plan from the operator tree
    Map<SemanticRule, SemanticNodeProcessor> opRules = new LinkedHashMap<SemanticRule, SemanticNodeProcessor>();
    opRules.put(new RuleRegExp("TS", TableScanOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getTableScanRule());
    opRules.put(new RuleRegExp("RS", ReduceSinkOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getReduceSinkRule());
    opRules.put(new RuleRegExp("JOIN", JoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getJoinRule());
    opRules.put(new RuleRegExp("MAPJOIN", MapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
    opRules.put(new RuleRegExp("SMB", SMBMapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
    opRules.put(new RuleRegExp("MUX", MuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
    opRules.put(new RuleRegExp("DEMUX", DemuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
    opRules.put(new RuleRegExp("UNION", UnionOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
    opRules.put(new RuleRegExp("GBY", GroupByOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getGroupByRule());
    opRules.put(new RuleRegExp("PTF", PTFOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getPTFRule());
    opRules.put(new RuleRegExp("SEL", SelectOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getSelectRule());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    SemanticDispatcher disp = new DefaultRuleDispatcher(OpTraitsRulesProcFactory.getDefaultRule(), opRules, annotateCtx);
    SemanticGraphWalker ogw = new LevelOrderWalker(disp, 0);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 38 with SemanticGraphWalker

use of org.apache.hadoop.hive.ql.lib.SemanticGraphWalker in project hive by apache.

the class PartitionConditionRemover method transform.

/*
   * (non-Javadoc)
   *
   * @see
   * org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop
   * .hive.ql.parse.ParseContext)
   */
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // create a the context for walking operators
    List<PcrOpWalkerCtx.OpToDeleteInfo> opToRemove = new ArrayList<PcrOpWalkerCtx.OpToDeleteInfo>();
    PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove);
    Map<SemanticRule, SemanticNodeProcessor> opRules = new LinkedHashMap<SemanticRule, SemanticNodeProcessor>();
    opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), PcrOpProcFactory.getFilterProc());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    SemanticDispatcher disp = new DefaultRuleDispatcher(PcrOpProcFactory.getDefaultProc(), opRules, opWalkerCtx);
    SemanticGraphWalker ogw = new DefaultGraphWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    for (PcrOpWalkerCtx.OpToDeleteInfo entry : opToRemove) {
        entry.getParent().removeChildAndAdoptItsChildren(entry.getOperator());
    }
    return pctx;
}

Example 39 with SemanticGraphWalker

use of org.apache.hadoop.hive.ql.lib.SemanticGraphWalker in project hive by apache.

the class ExprProcFactory method getExprDependency.

/**
 * Gets the expression dependencies for the expression.
 *
 * @param lctx
 *          The lineage context containing the input operators dependencies.
 * @param inpOp
 *          The input operator to the current operator.
 * @param expr
 *          The expression that is being processed.
 * @param outputMap
 * @throws SemanticException
 */
public static Dependency getExprDependency(LineageCtx lctx, Operator<? extends OperatorDesc> inpOp, ExprNodeDesc expr, HashMap<Node, Object> outputMap) throws SemanticException {
    outputMap.clear();
    // Create the walker, the rules dispatcher and the context.
    ExprProcCtx exprCtx = new ExprProcCtx(lctx, inpOp);
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    SemanticDispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, exprCtx);
    SemanticGraphWalker egw = new DefaultGraphWalker(disp);
    List<Node> startNodes = Collections.singletonList((Node) expr);
    egw.startWalking(startNodes, outputMap);
    return (Dependency) outputMap.get(expr);
}

Example 40 with SemanticGraphWalker

use of org.apache.hadoop.hive.ql.lib.SemanticGraphWalker in project hive by apache.

the class TezCompiler method markOperatorsWithUnstableRuntimeStats.

private void markOperatorsWithUnstableRuntimeStats(OptimizeTezProcContext procCtx) throws SemanticException {
    Map<SemanticRule, SemanticNodeProcessor> opRules = new LinkedHashMap<SemanticRule, SemanticNodeProcessor>();
    opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), new MarkRuntimeStatsAsIncorrect());
    opRules.put(new RuleRegExp("R2", AppMasterEventOperator.getOperatorName() + "%"), new MarkRuntimeStatsAsIncorrect());
    opRules.put(new RuleRegExp("R3", TableScanOperator.getOperatorName() + "%"), new MarkRuntimeStatsAsIncorrect());
    SemanticDispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(procCtx.parseContext.getTopOps().values());
    SemanticGraphWalker ogw = new PreOrderOnceWalker(disp);
    ogw.startWalking(topNodes, null);
}