Examples with WmContext org.apache.hadoop.hive.ql.wm.WmContext used on opensource projects

Example 1 with WmContext

use of org.apache.hadoop.hive.ql.wm.WmContext in project hive by apache.

the class PostExecWMEventsSummaryPrinter method run.

@Override
public void run(HookContext hookContext) throws Exception {
    assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK || hookContext.getHookType() == HookContext.HookType.ON_FAILURE_HOOK);
    HiveConf conf = hookContext.getConf();
    if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
        return;
    }
    LOG.info("Executing post execution hook to print workload manager events summary..");
    SessionState.LogHelper console = SessionState.getConsole();
    QueryPlan plan = hookContext.getQueryPlan();
    if (plan == null) {
        return;
    }
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        WmContext wmContext = tezTask.getDriverContext().getCtx().getWmContext();
        if (wmContext != null) {
            wmContext.shortPrint(console);
        }
    }
}

Example 2 with WmContext

use of org.apache.hadoop.hive.ql.wm.WmContext in project hive by apache.

the class TriggerValidatorRunnable method run.

@Override
public void run() {
    try {
        Map<TezSessionState, Trigger> violatedSessions = new HashMap<>();
        final List<TezSessionState> sessions = sessionTriggerProvider.getSessions();
        final List<Trigger> triggers = sessionTriggerProvider.getTriggers();
        for (TezSessionState sessionState : sessions) {
            WmContext wmContext = sessionState.getWmContext();
            if (wmContext != null && !wmContext.isQueryCompleted() && !wmContext.getSubscribedCounters().isEmpty()) {
                Map<String, Long> currentCounters = wmContext.getCurrentCounters();
                wmContext.updateElapsedTimeCounter();
                for (Trigger currentTrigger : triggers) {
                    String desiredCounter = currentTrigger.getExpression().getCounterLimit().getName();
                    // there could be interval where desired counter value is not populated by the time we make this check
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Validating trigger: {} against currentCounters: {}", currentTrigger, currentCounters);
                    }
                    if (currentCounters.containsKey(desiredCounter)) {
                        long currentCounterValue = currentCounters.get(desiredCounter);
                        if (currentTrigger.apply(currentCounterValue)) {
                            String queryId = sessionState.getWmContext().getQueryId();
                            if (violatedSessions.containsKey(sessionState)) {
                                // session already has a violation
                                Trigger existingTrigger = violatedSessions.get(sessionState);
                                // KILL always takes priority over MOVE
                                if (existingTrigger.getAction().getType().equals(Action.Type.MOVE_TO_POOL) && currentTrigger.getAction().getType().equals(Action.Type.KILL_QUERY)) {
                                    currentTrigger.setViolationMsg("Trigger " + currentTrigger + " violated. Current value: " + currentCounterValue);
                                    violatedSessions.put(sessionState, currentTrigger);
                                    LOG.info("KILL trigger replacing MOVE for query {}", queryId);
                                } else {
                                    // if multiple MOVE happens, only first move will be chosen
                                    LOG.warn("Conflicting MOVE triggers ({} and {}). Choosing the first MOVE trigger: {}", existingTrigger, currentTrigger, existingTrigger.getName());
                                }
                            } else {
                                // first violation for the session
                                currentTrigger.setViolationMsg("Trigger " + currentTrigger + " violated. Current value: " + currentCounterValue);
                                violatedSessions.put(sessionState, currentTrigger);
                            }
                        }
                    }
                }
                Trigger chosenTrigger = violatedSessions.get(sessionState);
                if (chosenTrigger != null) {
                    LOG.info("Query: {}. {}. Applying action.", sessionState.getWmContext().getQueryId(), chosenTrigger.getViolationMsg());
                }
            }
        }
        if (!violatedSessions.isEmpty()) {
            triggerActionHandler.applyAction(violatedSessions);
        }
    } catch (Throwable t) {
        // if exception is thrown in scheduled tasks, no further tasks will be scheduled, hence this ugly catch
        LOG.warn(TriggerValidatorRunnable.class.getSimpleName() + " caught exception.", t);
    }
}

Example 3 with WmContext

use of org.apache.hadoop.hive.ql.wm.WmContext in project hive by apache.

the class WorkloadManager method updateTriggers.

void updateTriggers(final WmTezSession session) {
    WmContext wmContext = session.getWmContext();
    String poolName = session.getPoolName();
    PoolState poolState = pools.get(poolName);
    if (wmContext != null && poolState != null) {
        wmContext.addTriggers(poolState.getTriggers());
        LOG.info("Subscribed to counters: {}", wmContext.getSubscribedCounters());
    }
}

Example 4 with WmContext

use of org.apache.hadoop.hive.ql.wm.WmContext in project hive by apache.

the class TezJobMonitor method monitorExecution.

public int monitorExecution() {
    boolean done = false;
    boolean success = false;
    int failedCounter = 0;
    final StopWatch failureTimer = new StopWatch();
    int rc = 0;
    DAGStatus status = null;
    Map<String, Progress> vertexProgressMap = null;
    long monitorStartTime = System.currentTimeMillis();
    synchronized (shutdownList) {
        shutdownList.add(dagClient);
    }
    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_DAG);
    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING);
    DAGStatus.State lastState = null;
    boolean running = false;
    long checkInterval = HiveConf.getTimeVar(hiveConf, HiveConf.ConfVars.TEZ_DAG_STATUS_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
    WmContext wmContext = null;
    while (true) {
        try {
            if (context != null) {
                context.checkHeartbeaterLockException();
            }
            status = dagClient.getDAGStatus(EnumSet.of(StatusGetOpts.GET_COUNTERS), checkInterval);
            TezCounters dagCounters = status.getDAGCounters();
            vertexProgressMap = status.getVertexProgress();
            wmContext = context.getWmContext();
            List<String> vertexNames = vertexProgressMap.keySet().stream().map(k -> k.replaceAll(" ", "_")).collect(Collectors.toList());
            if (dagCounters != null && wmContext != null) {
                Set<String> desiredCounters = wmContext.getSubscribedCounters();
                if (desiredCounters != null && !desiredCounters.isEmpty()) {
                    Map<String, Long> currentCounters = getCounterValues(dagCounters, vertexNames, vertexProgressMap, desiredCounters, done);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Requested DAG status. checkInterval: {}. currentCounters: {}", checkInterval, currentCounters);
                    }
                    wmContext.setCurrentCounters(currentCounters);
                }
            }
            DAGStatus.State state = status.getState();
            // AM is responsive again (recovery?)
            failedCounter = 0;
            failureTimer.reset();
            if (state != lastState || state == RUNNING) {
                lastState = state;
                switch(state) {
                    case SUBMITTED:
                        console.printInfo("Status: Submitted");
                        break;
                    case INITING:
                        console.printInfo("Status: Initializing");
                        this.executionStartTime = System.currentTimeMillis();
                        break;
                    case RUNNING:
                        if (!running) {
                            perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING);
                            console.printInfo("Status: Running (" + dagClient.getExecutionContext() + ")\n");
                            this.executionStartTime = System.currentTimeMillis();
                            running = true;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        break;
                    case SUCCEEDED:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        success = true;
                        running = false;
                        done = true;
                        break;
                    case KILLED:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        console.printInfo("Status: Killed");
                        running = false;
                        done = true;
                        rc = 1;
                        break;
                    case FAILED:
                    case ERROR:
                        if (!running) {
                            this.executionStartTime = monitorStartTime;
                        }
                        updateFunction.update(status, vertexProgressMap);
                        console.printError("Status: Failed");
                        running = false;
                        done = true;
                        rc = 2;
                        break;
                }
            }
            if (wmContext != null && done) {
                wmContext.setQueryCompleted(true);
            }
        } catch (Exception e) {
            console.printInfo("Exception: " + e.getMessage());
            boolean isInterrupted = hasInterruptedException(e);
            if (failedCounter == 0) {
                failureTimer.reset();
                failureTimer.start();
            }
            if (isInterrupted || (++failedCounter >= MAX_RETRY_FAILURES && failureTimer.now(TimeUnit.MILLISECONDS) > MAX_RETRY_INTERVAL)) {
                try {
                    if (isInterrupted) {
                        console.printInfo("Killing DAG...");
                    } else {
                        console.printInfo(String.format("Killing DAG... after %d seconds", failureTimer.now(TimeUnit.SECONDS)));
                    }
                    dagClient.tryKillDAG();
                } catch (IOException | TezException tezException) {
                // best effort
                }
                console.printError("Execution has failed. stack trace: " + ExceptionUtils.getStackTrace(e));
                rc = 1;
                done = true;
            } else {
                console.printInfo("Retrying...");
            }
            if (wmContext != null && done) {
                wmContext.setQueryCompleted(true);
            }
        } finally {
            if (done) {
                if (wmContext != null && done) {
                    wmContext.setQueryCompleted(true);
                }
                if (rc != 0 && status != null) {
                    for (String diag : status.getDiagnostics()) {
                        console.printError(diag);
                        diagnostics.append(diag);
                    }
                }
                synchronized (shutdownList) {
                    shutdownList.remove(dagClient);
                }
                break;
            }
        }
    }
    perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_RUN_DAG);
    printSummary(success, vertexProgressMap);
    return rc;
}

Example 5 with WmContext

use of org.apache.hadoop.hive.ql.wm.WmContext in project hive by apache.

the class Driver method setTriggerContext.

private void setTriggerContext(final String queryId) {
    final long queryStartTime;
    // the time when query display object is created)
    if (queryInfo != null) {
        queryStartTime = queryInfo.getBeginTime();
    } else {
        queryStartTime = queryDisplay.getQueryStartTime();
    }
    WmContext wmContext = new WmContext(queryStartTime, queryId);
    ctx.setWmContext(wmContext);
}