Examples with Trigger org.apache.hadoop.hive.ql.wm.Trigger used on opensource projects

Example 1 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TezSessionPoolManager method updateTriggers.

public void updateTriggers(final WMFullResourcePlan appliedRp) {
    if (sessionTriggerProvider != null) {
        List<WMTrigger> wmTriggers = appliedRp != null ? appliedRp.getTriggers() : null;
        List<Trigger> triggers = new ArrayList<>();
        if (wmTriggers != null) {
            for (WMTrigger wmTrigger : wmTriggers) {
                if (wmTrigger.isSetIsInUnmanaged() && wmTrigger.isIsInUnmanaged()) {
                    triggers.add(ExecutionTrigger.fromWMTrigger(wmTrigger));
                }
            }
        }
        sessionTriggerProvider.setTriggers(Collections.unmodifiableList(triggers));
    }
}

Example 2 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TriggerValidatorRunnable method run.

@Override
public void run() {
    try {
        Map<TezSessionState, Trigger> violatedSessions = new HashMap<>();
        final List<TezSessionState> sessions = sessionTriggerProvider.getSessions();
        final List<Trigger> triggers = sessionTriggerProvider.getTriggers();
        for (TezSessionState sessionState : sessions) {
            WmContext wmContext = sessionState.getWmContext();
            if (wmContext != null && !wmContext.isQueryCompleted() && !wmContext.getSubscribedCounters().isEmpty()) {
                Map<String, Long> currentCounters = wmContext.getCurrentCounters();
                wmContext.updateElapsedTimeCounter();
                for (Trigger currentTrigger : triggers) {
                    String desiredCounter = currentTrigger.getExpression().getCounterLimit().getName();
                    // there could be interval where desired counter value is not populated by the time we make this check
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Validating trigger: {} against currentCounters: {}", currentTrigger, currentCounters);
                    }
                    if (currentCounters.containsKey(desiredCounter)) {
                        long currentCounterValue = currentCounters.get(desiredCounter);
                        if (currentTrigger.apply(currentCounterValue)) {
                            String queryId = sessionState.getWmContext().getQueryId();
                            if (violatedSessions.containsKey(sessionState)) {
                                // session already has a violation
                                Trigger existingTrigger = violatedSessions.get(sessionState);
                                // KILL always takes priority over MOVE
                                if (existingTrigger.getAction().getType().equals(Action.Type.MOVE_TO_POOL) && currentTrigger.getAction().getType().equals(Action.Type.KILL_QUERY)) {
                                    currentTrigger.setViolationMsg("Trigger " + currentTrigger + " violated. Current value: " + currentCounterValue);
                                    violatedSessions.put(sessionState, currentTrigger);
                                    LOG.info("KILL trigger replacing MOVE for query {}", queryId);
                                } else {
                                    // if multiple MOVE happens, only first move will be chosen
                                    LOG.warn("Conflicting MOVE triggers ({} and {}). Choosing the first MOVE trigger: {}", existingTrigger, currentTrigger, existingTrigger.getName());
                                }
                            } else {
                                // first violation for the session
                                currentTrigger.setViolationMsg("Trigger " + currentTrigger + " violated. Current value: " + currentCounterValue);
                                violatedSessions.put(sessionState, currentTrigger);
                            }
                        }
                    }
                }
                Trigger chosenTrigger = violatedSessions.get(sessionState);
                if (chosenTrigger != null) {
                    LOG.info("Query: {}. {}. Applying action.", sessionState.getWmContext().getQueryId(), chosenTrigger.getViolationMsg());
                }
            }
        }
        if (!violatedSessions.isEmpty()) {
            triggerActionHandler.applyAction(violatedSessions);
        }
    } catch (Throwable t) {
        // if exception is thrown in scheduled tasks, no further tasks will be scheduled, hence this ugly catch
        LOG.warn(TriggerValidatorRunnable.class.getSimpleName() + " caught exception.", t);
    }
}

Example 3 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersNoTezSessionPool method testTriggerVertexTotalTasks.

@Test(timeout = 60000)
public void testTriggerVertexTotalTasks() throws Exception {
    Expression expression = ExpressionFactory.fromString("VERTEX_TOTAL_TASKS > 50");
    Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, getConfigs(), trigger + " violated");
}

Example 4 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersNoTezSessionPool method testTriggerTotalLaunchedTasks.

@Test(timeout = 60000)
public void testTriggerTotalLaunchedTasks() throws Exception {
    Expression expression = ExpressionFactory.fromString("TOTAL_LAUNCHED_TASKS > 50");
    Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, getConfigs(), trigger + " violated");
}

Example 5 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerDagTotalTasks.

@Test(timeout = 60000)
public void testTriggerDagTotalTasks() throws Exception {
    Expression expression = ExpressionFactory.fromString("DAG_TOTAL_TASKS > 50");
    Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, getConfigs(), trigger + " violated");
}