use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.
the class AnnotateWithOpTraits method transform.
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
AnnotateOpTraitsProcCtx annotateCtx = new AnnotateOpTraitsProcCtx(pctx);
// create a walker which walks the tree in a BFS manner while maintaining the
// operator stack. The dispatcher generates the plan from the operator tree
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("TS", TableScanOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getTableScanRule());
opRules.put(new RuleRegExp("RS", ReduceSinkOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getReduceSinkRule());
opRules.put(new RuleRegExp("JOIN", JoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getJoinRule());
opRules.put(new RuleRegExp("MAPJOIN", MapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
opRules.put(new RuleRegExp("SMB", SMBMapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
opRules.put(new RuleRegExp("MUX", MuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
opRules.put(new RuleRegExp("DEMUX", DemuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
opRules.put(new RuleRegExp("UNION", UnionOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule());
opRules.put(new RuleRegExp("GBY", GroupByOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getGroupByRule());
opRules.put(new RuleRegExp("SEL", SelectOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getSelectRule());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(OpTraitsRulesProcFactory.getDefaultRule(), opRules, annotateCtx);
GraphWalker ogw = new LevelOrderWalker(disp, 0);
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pctx.getTopOps().values());
ogw.startWalking(topNodes, null);
return pctx;
}
use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.
the class PartitionConditionRemover method transform.
/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop
* .hive.ql.parse.ParseContext)
*/
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
// create a the context for walking operators
List<PcrOpWalkerCtx.OpToDeleteInfo> opToRemove = new ArrayList<PcrOpWalkerCtx.OpToDeleteInfo>();
PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove);
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), PcrOpProcFactory.getFilterProc());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(PcrOpProcFactory.getDefaultProc(), opRules, opWalkerCtx);
GraphWalker ogw = new DefaultGraphWalker(disp);
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pctx.getTopOps().values());
ogw.startWalking(topNodes, null);
for (PcrOpWalkerCtx.OpToDeleteInfo entry : opToRemove) {
entry.getParent().removeChildAndAdoptItsChildren(entry.getOperator());
}
return pctx;
}
use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.
the class GenTezWorkWalker method startWalking.
/**
* starting point for walking.
*
* @throws SemanticException
*/
@Override
public void startWalking(Collection<Node> startNodes, HashMap<Node, Object> nodeOutput) throws SemanticException {
toWalk.addAll(startNodes);
while (toWalk.size() > 0) {
Node nd = toWalk.remove(0);
setRoot(nd);
walk(nd);
if (nodeOutput != null) {
nodeOutput.put(nd, retMap.get(nd));
}
}
}
use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.
the class MapReduceCompiler method generateTaskTree.
@Override
protected void generateTaskTree(List<Task<? extends Serializable>> rootTasks, ParseContext pCtx, List<Task<MoveWork>> mvTask, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException {
// generate map reduce plans
ParseContext tempParseContext = getParseContext(pCtx, rootTasks);
GenMRProcContext procCtx = new GenMRProcContext(conf, // Must be deterministic order map for consistent q-test output across Java versions
new LinkedHashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>>(), tempParseContext, mvTask, rootTasks, new LinkedHashMap<Operator<? extends OperatorDesc>, GenMapRedCtx>(), inputs, outputs);
// create a walker which walks the tree in a DFS manner while maintaining
// the operator stack.
// The dispatcher generates the plan from the operator tree
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp(new String("R1"), TableScanOperator.getOperatorName() + "%"), new GenMRTableScan1());
opRules.put(new RuleRegExp(new String("R2"), TableScanOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink1());
opRules.put(new RuleRegExp(new String("R3"), ReduceSinkOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink2());
opRules.put(new RuleRegExp(new String("R4"), FileSinkOperator.getOperatorName() + "%"), new GenMRFileSink1());
opRules.put(new RuleRegExp(new String("R5"), UnionOperator.getOperatorName() + "%"), new GenMRUnion1());
opRules.put(new RuleRegExp(new String("R6"), UnionOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink3());
opRules.put(new RuleRegExp(new String("R7"), MapJoinOperator.getOperatorName() + "%"), MapJoinFactory.getTableScanMapJoin());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules, procCtx);
GraphWalker ogw = new GenMapRedWalker(disp);
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(pCtx.getTopOps().values());
ogw.startWalking(topNodes, null);
}
use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.
the class GenMapRedWalker method walk.
/**
* Walk the given operator.
*
* @param nd
* operator being walked
*/
@Override
protected void walk(Node nd) throws SemanticException {
List<? extends Node> children = nd.getChildren();
// maintain the stack of operators encountered
opStack.push(nd);
Boolean result = dispatchAndReturn(nd, opStack);
// need not be traversed again
if (children == null || result == Boolean.FALSE) {
opStack.pop();
return;
}
// move all the children to the front of queue
for (Node ch : children) {
walk(ch);
}
// done with this operator
opStack.pop();
}
Aggregations