Examples with NodeProcessor org.apache.hadoop.hive.ql.lib.NodeProcessor used on opensource projects

Example 46 with NodeProcessor

use of org.apache.hadoop.hive.ql.lib.NodeProcessor in project hive by apache.

the class SortedMergeBucketMapJoinOptimizer method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    HiveConf conf = pctx.getConf();
    SortBucketJoinProcCtx smbJoinContext = new SortBucketJoinProcCtx(conf);
    // Get a list of joins which cannot be converted to a sort merge join
    // Only selects and filters operators are allowed between the table scan and
    // join currently. More operators can be added - the method supportAutomaticSortMergeJoin
    // dictates which operator is allowed
    getListOfRejectedJoins(pctx, smbJoinContext);
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    // go through all map joins and find out all which have enabled bucket map
    // join.
    opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + "%"), getSortedMergeBucketMapjoinProc(pctx));
    // converted to sort-merge join
    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN)) {
        opRules.put(new RuleRegExp("R2", "JOIN%"), getSortedMergeJoinProc(pctx));
    }
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, smbJoinContext);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 47 with NodeProcessor

use of org.apache.hadoop.hive.ql.lib.NodeProcessor in project hive by apache.

the class SortedMergeBucketMapJoinOptimizer method getCheckCandidateJoin.

// check if the join operator encountered is a candidate for being converted
// to a sort-merge join
private NodeProcessor getCheckCandidateJoin() {
    return new NodeProcessor() {

        @Override
        public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException {
            SortBucketJoinProcCtx smbJoinContext = (SortBucketJoinProcCtx) procCtx;
            JoinOperator joinOperator = (JoinOperator) nd;
            int size = stack.size();
            if (!(stack.get(size - 1) instanceof JoinOperator) || !(stack.get(size - 2) instanceof ReduceSinkOperator)) {
                smbJoinContext.getRejectedJoinOps().add(joinOperator);
                return null;
            }
            // not be converted.
            for (int pos = size - 3; pos >= 0; pos--) {
                Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) stack.get(pos);
                if (!op.supportAutomaticSortMergeJoin()) {
                    smbJoinContext.getRejectedJoinOps().add(joinOperator);
                    return null;
                }
            }
            return null;
        }
    };
}

Example 48 with NodeProcessor

use of org.apache.hadoop.hive.ql.lib.NodeProcessor in project hive by apache.

the class PartitionColumnsSeparator method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // 1. Trigger transformation
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), new StructInTransformer());
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new ForwardWalker(disp);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 49 with NodeProcessor

use of org.apache.hadoop.hive.ql.lib.NodeProcessor in project hive by apache.

the class HiveOpConverterPostProc method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // 0. We check the conditions to apply this transformation,
    // if we do not meet them we bail out
    final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED);
    final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
    final boolean cboSucceeded = pctx.getContext().isCboSucceeded();
    if (!(cboEnabled && returnPathEnabled && cboSucceeded)) {
        return pctx;
    }
    // 1. Initialize aux data structures
    this.pctx = pctx;
    this.aliasToOpInfo = new HashMap<String, Operator<? extends OperatorDesc>>();
    // 2. Trigger transformation
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", JoinOperator.getOperatorName() + "%"), new JoinAnnotate());
    opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + "%"), new TableScanAnnotate());
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null);
    GraphWalker ogw = new ForwardWalker(disp);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 50 with NodeProcessor

use of org.apache.hadoop.hive.ql.lib.NodeProcessor in project hive by apache.

the class ColumnPruner method transform.

/**
 * Transform the query tree. For each table under consideration, check if all
 * columns are needed. If not, only select the operators needed at the
 * beginning and proceed.
 *
 * @param pactx
 *          the current parse context
 */
@Override
public ParseContext transform(ParseContext pactx) throws SemanticException {
    pGraphContext = pactx;
    // generate pruned column list for all relevant operators
    ColumnPrunerProcCtx cppCtx = new ColumnPrunerProcCtx(pactx);
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getFilterProc());
    opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getGroupByProc());
    opRules.put(new RuleRegExp("R3", ReduceSinkOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getReduceSinkProc());
    opRules.put(new RuleRegExp("R4", SelectOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getSelectProc());
    opRules.put(new RuleRegExp("R5", CommonJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getJoinProc());
    opRules.put(new RuleRegExp("R6", MapJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getMapJoinProc());
    opRules.put(new RuleRegExp("R7", TableScanOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getTableScanProc());
    opRules.put(new RuleRegExp("R8", LateralViewJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLateralViewJoinProc());
    opRules.put(new RuleRegExp("R9", LateralViewForwardOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLateralViewForwardProc());
    opRules.put(new RuleRegExp("R10", PTFOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getPTFProc());
    opRules.put(new RuleRegExp("R11", ScriptOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getScriptProc());
    opRules.put(new RuleRegExp("R12", LimitOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLimitProc());
    opRules.put(new RuleRegExp("R13", UnionOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getUnionProc());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory.getDefaultProc(), opRules, cppCtx);
    GraphWalker ogw = new ColumnPrunerWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pGraphContext.getTopOps().values());
    ogw.startWalking(topNodes, null);
    // set it back so that column pruner in the optimizer will not do the
    // view column authorization again even if it is triggered again.
    pGraphContext.setNeedViewColumnAuthorization(false);
    return pGraphContext;
}