Examples with SessionTriggerProvider org.apache.hadoop.hive.ql.wm.SessionTriggerProvider used on opensource projects

Example 1 with SessionTriggerProvider

use of org.apache.hadoop.hive.ql.wm.SessionTriggerProvider in project hive by apache.

the class PerPoolTriggerValidatorRunnable method run.

@Override
public void run() {
    try {
        ScheduledExecutorService validatorExecutorService = Executors.newScheduledThreadPool(sessionTriggerProviders.size());
        for (Map.Entry<String, SessionTriggerProvider> entry : sessionTriggerProviders.entrySet()) {
            String poolName = entry.getKey();
            if (!poolValidators.containsKey(poolName)) {
                LOG.info("Creating trigger validator for pool: {}", poolName);
                TriggerValidatorRunnable poolValidator = new TriggerValidatorRunnable(entry.getValue(), triggerActionHandler);
                validatorExecutorService.scheduleWithFixedDelay(poolValidator, triggerValidationIntervalMs, triggerValidationIntervalMs, TimeUnit.MILLISECONDS);
                poolValidators.put(poolName, poolValidator);
            }
        }
    } catch (Throwable t) {
        // if exception is thrown in scheduled tasks, no further tasks will be scheduled, hence this ugly catch
        LOG.warn(PerPoolTriggerValidatorRunnable.class.getSimpleName() + " caught exception.", t);
    }
}

Example 2 with SessionTriggerProvider

use of org.apache.hadoop.hive.ql.wm.SessionTriggerProvider in project hive by apache.

the class TezSessionPoolManager method initTriggers.

public void initTriggers(final HiveConf conf) {
    if (triggerValidatorRunnable == null) {
        final long triggerValidationIntervalMs = HiveConf.getTimeVar(conf, ConfVars.HIVE_TRIGGER_VALIDATION_INTERVAL, TimeUnit.MILLISECONDS);
        sessionTriggerProvider = new SessionTriggerProvider(openSessions, new LinkedList<>());
        triggerActionHandler = new KillTriggerActionHandler();
        triggerValidatorRunnable = new TriggerValidatorRunnable(sessionTriggerProvider, triggerActionHandler);
        startTriggerValidator(triggerValidationIntervalMs);
    }
}

Example 3 with SessionTriggerProvider

use of org.apache.hadoop.hive.ql.wm.SessionTriggerProvider in project hive by apache.

the class TestWorkloadManager method testMoveSessionsMultiPool.

@Test(timeout = 10000)
public void testMoveSessionsMultiPool() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 0.4f), pool("B", 1, 0.4f), pool("B.x", 1, 0.2f), pool("B.y", 1, 0.8f), pool("C", 1, 0.2f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B"), mapping("C", "C")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0, B.x: 0, B.y: 0, C: 0]
    Map<String, SessionTriggerProvider> allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertEquals("A", sessionA1.getPoolName());
    // [A: 0, B: 1, B.x: 0, B.y: 0, C: 0]
    Future<Boolean> future = wm.applyMoveSessionAsync(sessionA1, "B.y");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.32f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("B.y").getSessions().contains(sessionA1));
    assertEquals("B.y", sessionA1.getPoolName());
    // [A: 0, B: 0, B.x: 0, B.y: 0, C: 1]
    future = wm.applyMoveSessionAsync(sessionA1, "C");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(1, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("C").getSessions().contains(sessionA1));
    assertEquals("C", sessionA1.getPoolName());
    // [A: 0, B: 0, B.x: 1, B.y: 0, C: 0]
    future = wm.applyMoveSessionAsync(sessionA1, "B.x");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.08f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("B.x").getSessions().contains(sessionA1));
    assertEquals("B.x", sessionA1.getPoolName());
    WmTezSession sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0, B.x: 1, B.y: 0, C: 0]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.4f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.08f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B.x").getSessions().contains(sessionA1));
    assertEquals("A", sessionA2.getPoolName());
    assertEquals("B.x", sessionA1.getPoolName());
    // A is maxed out on capacity, so this move should fail the session
    // [A: 1, B: 0, B.x: 0, B.y: 0, C: 0]
    future = wm.applyMoveSessionAsync(sessionA1, "A");
    assertNotNull(future.get());
    assertFalse(future.get());
    wm.addTestEvent().get();
    while (sessionA1.isOpen()) {
        Thread.sleep(100);
    }
    assertNull(sessionA1.getPoolName());
    assertEquals("Destination pool A is full. Killing query.", sessionA1.getReasonForKill());
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    // return a loaned session goes back to tez am pool
    // [A: 0, B: 0, B.x: 0, B.y: 0, C: 0]
    wm.returnAfterUse(sessionA2);
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.0f, sessionA1.getClusterFraction(), EPSILON);
    assertFalse(allSessionProviders.get("A").getSessions().contains(sessionA1));
}

Example 4 with SessionTriggerProvider

use of org.apache.hadoop.hive.ql.wm.SessionTriggerProvider in project hive by apache.

the class WorkloadManager method updateSessionTriggerProvidersOnMasterThread.

private void updateSessionTriggerProvidersOnMasterThread() {
    for (Map.Entry<String, PoolState> entry : pools.entrySet()) {
        String poolName = entry.getKey();
        PoolState poolState = entry.getValue();
        final List<Trigger> triggers = Collections.unmodifiableList(poolState.getTriggers());
        final List<TezSessionState> sessionStates = Collections.unmodifiableList(poolState.getSessions());
        SessionTriggerProvider sessionTriggerProvider = perPoolProviders.get(poolName);
        if (sessionTriggerProvider != null) {
            perPoolProviders.get(poolName).setTriggers(triggers);
            perPoolProviders.get(poolName).setSessions(sessionStates);
        } else {
            perPoolProviders.put(poolName, new SessionTriggerProvider(sessionStates, triggers));
        }
    }
}

Example 5 with SessionTriggerProvider

use of org.apache.hadoop.hive.ql.wm.SessionTriggerProvider in project hive by apache.

the class TestWorkloadManager method testMoveSessions.

@Test(timeout = 10000)
public void testMoveSessions() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 0.6f), pool("B", 2, 0.4f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0]
    Map<String, SessionTriggerProvider> allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertFalse(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA1.getPoolName());
    // [A: 0, B: 1]
    Future<Boolean> future = wm.applyMoveSessionAsync(sessionA1, "B");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(1, allSessionProviders.get("B").getSessions().size());
    assertFalse(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("B", sessionA1.getPoolName());
    WmTezSession sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 1]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(1, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    // [A: 0, B: 2]
    future = wm.applyMoveSessionAsync(sessionA2, "B");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.2f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("B", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    WmTezSession sessionA3 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 2]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA3));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA3.getPoolName());
    assertEquals("B", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    // B is maxed out on capacity, so this move should fail the session
    future = wm.applyMoveSessionAsync(sessionA3, "B");
    assertNotNull(future.get());
    assertFalse(future.get());
    wm.addTestEvent().get();
    while (sessionA3.isOpen()) {
        Thread.sleep(100);
    }
    assertNull(sessionA3.getPoolName());
    assertEquals("Destination pool B is full. Killing query.", sessionA3.getReasonForKill());
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
}