Examples with Trigger org.apache.hadoop.hive.ql.wm.Trigger used on opensource projects

Example 36 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class WorkloadManager method applyNewResourcePlanOnMasterThread.

private void applyNewResourcePlanOnMasterThread(EventState e, WmThreadSyncWork syncWork, HashSet<String> poolsToRedistribute) {
    int totalQueryParallelism = 0;
    WMFullResourcePlan plan = e.resourcePlanToApply;
    if (plan == null) {
        // NULL plan means WM is disabled via a command; it could still be reenabled.
        LOG.info("Disabling workload management because the resource plan has been removed");
        this.rpName = null;
        this.defaultPool = null;
        this.userPoolMapping = new UserPoolMapping(null, null);
    } else {
        this.rpName = plan.getPlan().getName();
        this.defaultPool = plan.getPlan().getDefaultPoolPath();
        this.userPoolMapping = new UserPoolMapping(plan.getMappings(), defaultPool);
    }
    // Note: we assume here that plan has been validated beforehand, so we don't verify
    // that fractions or query parallelism add up, etc.
    Map<String, PoolState> oldPools = pools;
    pools = new HashMap<>();
    ArrayList<List<WMPool>> poolsByLevel = new ArrayList<>();
    if (plan != null) {
        // first distribute them by levels, then add level by level.
        for (WMPool pool : plan.getPools()) {
            String fullName = pool.getPoolPath();
            int ix = StringUtils.countMatches(fullName, POOL_SEPARATOR_STR);
            while (poolsByLevel.size() <= ix) {
                // We expect all the levels to have items.
                poolsByLevel.add(new LinkedList<WMPool>());
            }
            poolsByLevel.get(ix).add(pool);
        }
    }
    for (int level = 0; level < poolsByLevel.size(); ++level) {
        List<WMPool> poolsOnLevel = poolsByLevel.get(level);
        for (WMPool pool : poolsOnLevel) {
            String fullName = pool.getPoolPath();
            int qp = pool.getQueryParallelism();
            double fraction = pool.getAllocFraction();
            if (level > 0) {
                String parentName = fullName.substring(0, fullName.lastIndexOf(POOL_SEPARATOR));
                PoolState parent = pools.get(parentName);
                fraction = parent.finalFraction * fraction;
                parent.finalFractionRemaining -= fraction;
            }
            PoolState state = oldPools == null ? null : oldPools.remove(fullName);
            if (state == null) {
                state = new PoolState(fullName, qp, fraction, pool.getSchedulingPolicy());
            } else {
                // This will also take care of the queries if query parallelism changed.
                state.update(qp, fraction, syncWork, e, pool.getSchedulingPolicy());
                poolsToRedistribute.add(fullName);
            }
            state.setTriggers(new LinkedList<Trigger>());
            LOG.info("Adding Hive pool: " + state);
            pools.put(fullName, state);
            totalQueryParallelism += qp;
        }
    }
    // GLOBAL - all pools inherit
    if (plan != null && plan.isSetTriggers() && plan.isSetPoolTriggers()) {
        Map<String, Trigger> triggers = new HashMap<>();
        for (WMTrigger trigger : plan.getTriggers()) {
            ExecutionTrigger execTrigger = ExecutionTrigger.fromWMTrigger(trigger);
            triggers.put(trigger.getTriggerName(), execTrigger);
        }
        for (WMPoolTrigger poolTrigger : plan.getPoolTriggers()) {
            PoolState pool = pools.get(poolTrigger.getPool());
            Trigger trigger = triggers.get(poolTrigger.getTrigger());
            pool.triggers.add(trigger);
            poolsToRedistribute.add(pool.fullName);
            LOG.info("Adding pool " + pool.fullName + " trigger " + trigger);
        }
    }
    if (oldPools != null && !oldPools.isEmpty()) {
        // Looks like some pools were removed; kill running queries, re-queue the queued ones.
        for (PoolState oldPool : oldPools.values()) {
            oldPool.destroy(syncWork, e.getRequests, e.toReuse);
        }
    }
    LOG.info("Updating with " + totalQueryParallelism + " total query parallelism");
    int deltaSessions = totalQueryParallelism - this.totalQueryParallelism;
    this.totalQueryParallelism = totalQueryParallelism;
    // Nothing to do.
    if (deltaSessions == 0)
        return;
    if (deltaSessions < 0) {
        // First, see if we have sessions that we were planning to restart/kill; get rid of those.
        deltaSessions = transferSessionsToDestroy(syncWork.toKillQuery.keySet(), syncWork.toDestroyNoRestart, deltaSessions);
        deltaSessions = transferSessionsToDestroy(syncWork.toRestartInUse, syncWork.toDestroyNoRestart, deltaSessions);
    }
    if (deltaSessions != 0) {
        failOnFutureFailure(tezAmPool.resizeAsync(deltaSessions, syncWork.toDestroyNoRestart));
    }
}

Example 37 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class WorkloadManager method updateSessionTriggerProvidersOnMasterThread.

private void updateSessionTriggerProvidersOnMasterThread() {
    for (Map.Entry<String, PoolState> entry : pools.entrySet()) {
        String poolName = entry.getKey();
        PoolState poolState = entry.getValue();
        final List<Trigger> triggers = Collections.unmodifiableList(poolState.getTriggers());
        final List<TezSessionState> sessionStates = Collections.unmodifiableList(poolState.getSessions());
        SessionTriggerProvider sessionTriggerProvider = perPoolProviders.get(poolName);
        if (sessionTriggerProvider != null) {
            perPoolProviders.get(poolName).setTriggers(triggers);
            perPoolProviders.get(poolName).setSessions(sessionStates);
        } else {
            perPoolProviders.put(poolName, new SessionTriggerProvider(sessionStates, triggers));
        }
    }
}