use of org.apache.hadoop.hive.ql.exec.tez.TezTask in project hive by apache.
the class OperatorStatsReaderHook method run.
@Override
public void run(HookContext hookContext) throws Exception {
if (hookContext.getHookType() == HookType.PRE_EXEC_HOOK) {
return;
}
if (hookContext.getHookType() == HookType.POST_EXEC_HOOK && !isCollectOnSuccess()) {
return;
}
HiveConf conf = hookContext.getConf();
QueryPlan plan = hookContext.getQueryPlan();
List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
for (TezTask tezTask : rootTasks) {
List<BaseWork> baseWorks = tezTask.getWork().getAllWork();
for (BaseWork baseWork : baseWorks) {
String vertexName = baseWork.getName();
LOG.debug("Reading runtime statistics for tez vertex task: {}", vertexName);
TezCounters counters = tezTask.getTezCounters();
if (counters != null) {
String groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
for (Operator<? extends OperatorDesc> op : baseWork.getAllOperators()) {
String operatorId = op.getOperatorId();
OperatorStats operatorStats = null;
String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId;
TezCounter tezCounter = counters.getGroup(groupName).findCounter(counterName, false);
if (tezCounter != null) {
if (operatorStats == null) {
operatorStats = new OperatorStats(operatorId);
}
operatorStats.setOutputRecords(tezCounter.getValue());
}
if (operatorStats != null) {
((PrivateHookContext) hookContext).getContext().getPlanMapper().link(op, operatorStats);
} else {
LOG.debug("Unable to get statistics for vertex: {} opId: {} groupName: {}", vertexName, operatorId, groupName);
}
}
}
}
}
}
use of org.apache.hadoop.hive.ql.exec.tez.TezTask in project hive by apache.
the class DDLTask method mergeFiles.
/**
* First, make sure the source table/partition is not
* archived/indexes/non-rcfile. If either of these is true, throw an
* exception.
*
* The way how it does the merge is to create a BlockMergeTask from the
* mergeFilesDesc.
*
* @param db
* @param mergeFilesDesc
* @return
* @throws HiveException
*/
private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, DriverContext driverContext) throws HiveException {
ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx();
boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir();
int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
// merge work only needs input and output.
MergeFileWork mergeWork = new MergeFileWork(mergeFilesDesc.getInputDir(), mergeFilesDesc.getOutputDir(), mergeFilesDesc.getInputFormatClass().getName(), mergeFilesDesc.getTableDesc());
LinkedHashMap<Path, ArrayList<String>> pathToAliases = new LinkedHashMap<>();
ArrayList<String> inputDirstr = new ArrayList<String>(1);
inputDirstr.add(mergeFilesDesc.getInputDir().toString());
pathToAliases.put(mergeFilesDesc.getInputDir().get(0), inputDirstr);
mergeWork.setPathToAliases(pathToAliases);
mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
mergeWork.resolveConcatenateMerge(db.getConf());
mergeWork.setMapperCannotSpanPartns(true);
mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass().getName());
final FileMergeDesc fmd;
if (mergeFilesDesc.getInputFormatClass().equals(RCFileInputFormat.class)) {
fmd = new RCFileMergeDesc();
} else {
// safe to assume else is ORC as semantic analyzer will check for RC/ORC
fmd = new OrcFileMergeDesc();
}
fmd.setDpCtx(null);
fmd.setHasDynamicPartitions(false);
fmd.setListBucketingAlterTableConcatenate(lbatc);
fmd.setListBucketingDepth(lbd);
fmd.setOutputPath(mergeFilesDesc.getOutputDir());
CompilationOpContext opContext = driverContext.getCtx().getOpContext();
Operator<? extends OperatorDesc> mergeOp = OperatorFactory.get(opContext, fmd);
LinkedHashMap<String, Operator<? extends OperatorDesc>> aliasToWork = new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
aliasToWork.put(mergeFilesDesc.getInputDir().toString(), mergeOp);
mergeWork.setAliasToWork(aliasToWork);
DriverContext driverCxt = new DriverContext();
Task<?> task;
if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
TezWork tezWork = new TezWork(queryState.getQueryId(), conf);
mergeWork.setName("File Merge");
tezWork.add(mergeWork);
task = new TezTask();
((TezTask) task).setWork(tezWork);
} else {
task = new MergeFileTask();
((MergeFileTask) task).setWork(mergeWork);
}
// initialize the task and execute
task.initialize(queryState, getQueryPlan(), driverCxt, opContext);
subtask = task;
int ret = task.execute(driverCxt);
if (subtask.getException() != null) {
setException(subtask.getException());
}
return ret;
}
use of org.apache.hadoop.hive.ql.exec.tez.TezTask in project hive by apache.
the class CrossProductHandler method dispatch.
@Override
public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) throws SemanticException {
@SuppressWarnings("unchecked") Task<? extends Serializable> currTask = (Task<? extends Serializable>) nd;
if (currTask instanceof MapRedTask) {
MapRedTask mrTsk = (MapRedTask) currTask;
MapredWork mrWrk = mrTsk.getWork();
checkMapJoins(mrTsk);
checkMRReducer(currTask.toString(), mrWrk);
} else if (currTask instanceof ConditionalTask) {
List<Task<? extends Serializable>> taskListInConditionalTask = ((ConditionalTask) currTask).getListTasks();
for (Task<? extends Serializable> tsk : taskListInConditionalTask) {
dispatch(tsk, stack, nodeOutputs);
}
} else if (currTask instanceof TezTask) {
TezTask tezTask = (TezTask) currTask;
TezWork tezWork = tezTask.getWork();
checkMapJoins(tezWork);
checkTezReducer(tezWork);
}
return null;
}
Aggregations