Examples with WMFullResourcePlan org.apache.hadoop.hive.metastore.api.WMFullResourcePlan used on opensource projects

Example 16 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestTriggersWorkloadManager method setupTriggers.

@Override
protected void setupTriggers(final List<Trigger> triggers) throws Exception {
    WorkloadManager wm = WorkloadManager.getInstance();
    WMPool pool = new WMPool("rp", "llap");
    pool.setAllocFraction(1.0f);
    pool.setQueryParallelism(1);
    WMFullResourcePlan rp = new WMFullResourcePlan(new WMResourcePlan("rp"), Lists.newArrayList(pool));
    rp.getPlan().setDefaultPoolPath("llap");
    for (Trigger trigger : triggers) {
        rp.addToTriggers(wmTriggerFromTrigger(trigger));
        rp.addToPoolTriggers(new WMPoolTrigger("llap", trigger.getName()));
    }
    wm.updateResourcePlanAsync(rp).get(10, TimeUnit.SECONDS);
}

Example 17 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class WorkloadManager method applyNewResourcePlanOnMasterThread.

private void applyNewResourcePlanOnMasterThread(EventState e, WmThreadSyncWork syncWork, HashSet<String> poolsToRedistribute) {
    int totalQueryParallelism = 0;
    WMFullResourcePlan plan = e.resourcePlanToApply;
    if (plan == null) {
        // NULL plan means WM is disabled via a command; it could still be reenabled.
        LOG.info("Disabling workload management because the resource plan has been removed");
        this.rpName = null;
        this.defaultPool = null;
        this.userPoolMapping = new UserPoolMapping(null, null);
    } else {
        this.rpName = plan.getPlan().getName();
        this.defaultPool = plan.getPlan().getDefaultPoolPath();
        this.userPoolMapping = new UserPoolMapping(plan.getMappings(), defaultPool);
    }
    // Note: we assume here that plan has been validated beforehand, so we don't verify
    // that fractions or query parallelism add up, etc.
    Map<String, PoolState> oldPools = pools;
    pools = new HashMap<>();
    ArrayList<List<WMPool>> poolsByLevel = new ArrayList<>();
    if (plan != null) {
        // first distribute them by levels, then add level by level.
        for (WMPool pool : plan.getPools()) {
            String fullName = pool.getPoolPath();
            int ix = StringUtils.countMatches(fullName, POOL_SEPARATOR_STR);
            while (poolsByLevel.size() <= ix) {
                // We expect all the levels to have items.
                poolsByLevel.add(new LinkedList<WMPool>());
            }
            poolsByLevel.get(ix).add(pool);
        }
    }
    for (int level = 0; level < poolsByLevel.size(); ++level) {
        List<WMPool> poolsOnLevel = poolsByLevel.get(level);
        for (WMPool pool : poolsOnLevel) {
            String fullName = pool.getPoolPath();
            int qp = pool.getQueryParallelism();
            double fraction = pool.getAllocFraction();
            if (level > 0) {
                String parentName = fullName.substring(0, fullName.lastIndexOf(POOL_SEPARATOR));
                PoolState parent = pools.get(parentName);
                fraction = parent.finalFraction * fraction;
                parent.finalFractionRemaining -= fraction;
            }
            PoolState state = oldPools == null ? null : oldPools.remove(fullName);
            if (state == null) {
                state = new PoolState(fullName, qp, fraction, pool.getSchedulingPolicy());
            } else {
                // This will also take care of the queries if query parallelism changed.
                state.update(qp, fraction, syncWork, e, pool.getSchedulingPolicy());
                poolsToRedistribute.add(fullName);
            }
            state.setTriggers(new LinkedList<Trigger>());
            LOG.info("Adding Hive pool: " + state);
            pools.put(fullName, state);
            totalQueryParallelism += qp;
        }
    }
    // GLOBAL - all pools inherit
    if (plan != null && plan.isSetTriggers() && plan.isSetPoolTriggers()) {
        Map<String, Trigger> triggers = new HashMap<>();
        for (WMTrigger trigger : plan.getTriggers()) {
            ExecutionTrigger execTrigger = ExecutionTrigger.fromWMTrigger(trigger);
            triggers.put(trigger.getTriggerName(), execTrigger);
        }
        for (WMPoolTrigger poolTrigger : plan.getPoolTriggers()) {
            PoolState pool = pools.get(poolTrigger.getPool());
            Trigger trigger = triggers.get(poolTrigger.getTrigger());
            pool.triggers.add(trigger);
            poolsToRedistribute.add(pool.fullName);
            LOG.info("Adding pool " + pool.fullName + " trigger " + trigger);
        }
    }
    if (oldPools != null && !oldPools.isEmpty()) {
        // Looks like some pools were removed; kill running queries, re-queue the queued ones.
        for (PoolState oldPool : oldPools.values()) {
            oldPool.destroy(syncWork, e.getRequests, e.toReuse);
        }
    }
    LOG.info("Updating with " + totalQueryParallelism + " total query parallelism");
    int deltaSessions = totalQueryParallelism - this.totalQueryParallelism;
    this.totalQueryParallelism = totalQueryParallelism;
    // Nothing to do.
    if (deltaSessions == 0)
        return;
    if (deltaSessions < 0) {
        // First, see if we have sessions that we were planning to restart/kill; get rid of those.
        deltaSessions = transferSessionsToDestroy(syncWork.toKillQuery.keySet(), syncWork.toDestroyNoRestart, deltaSessions);
        deltaSessions = transferSessionsToDestroy(syncWork.toRestartInUse, syncWork.toDestroyNoRestart, deltaSessions);
    }
    if (deltaSessions != 0) {
        failOnFutureFailure(tezAmPool.resizeAsync(deltaSessions, syncWork.toDestroyNoRestart));
    }
}

Example 18 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testMoveSessions.

@Test(timeout = 10000)
public void testMoveSessions() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 0.6f), pool("B", 2, 0.4f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0]
    Map<String, SessionTriggerProvider> allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertFalse(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA1.getPoolName());
    // [A: 0, B: 1]
    Future<Boolean> future = wm.applyMoveSessionAsync(sessionA1, "B");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(1, allSessionProviders.get("B").getSessions().size());
    assertFalse(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("B", sessionA1.getPoolName());
    WmTezSession sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 1]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(1, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    // [A: 0, B: 2]
    future = wm.applyMoveSessionAsync(sessionA2, "B");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.2f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("B", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    WmTezSession sessionA3 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 2]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA3));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B").getSessions().contains(sessionA1));
    assertEquals(0.6f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA3.getPoolName());
    assertEquals("B", sessionA2.getPoolName());
    assertEquals("B", sessionA1.getPoolName());
    // B is maxed out on capacity, so this move should fail the session
    future = wm.applyMoveSessionAsync(sessionA3, "B");
    assertNotNull(future.get());
    assertFalse(future.get());
    wm.addTestEvent().get();
    while (sessionA3.isOpen()) {
        Thread.sleep(100);
    }
    assertNull(sessionA3.getPoolName());
    assertEquals("Destination pool B is full. Killing query.", sessionA3.getReasonForKill());
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(2, allSessionProviders.get("B").getSessions().size());
}

Example 19 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testReuseWithDifferentPool.

@Test(timeout = 10000)
public void testReuseWithDifferentPool() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 2, 0.6f), pool("B", 1, 0.4f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf), sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    assertEquals("A", sessionA1.getPoolName());
    assertEquals(0.3f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA2.getPoolName());
    assertEquals(0.3f, sessionA2.getClusterFraction(), EPSILON);
    WmTezSession sessionB1 = (WmTezSession) wm.getSession(sessionA1, mappingInput("B"), conf);
    assertSame(sessionA1, sessionB1);
    assertEquals("B", sessionB1.getPoolName());
    assertEquals(0.4f, sessionB1.getClusterFraction(), EPSILON);
    // A1 removed from A.
    assertEquals(0.6f, sessionA2.getClusterFraction(), EPSILON);
    // Make sure that we can still get a session from A.
    WmTezSession sessionA3 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    assertEquals("A", sessionA3.getPoolName());
    assertEquals(0.3f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals(0.3f, sessionA3.getClusterFraction(), EPSILON);
    sessionA3.returnToSessionManager();
    sessionB1.returnToSessionManager();
    sessionA2.returnToSessionManager();
}

Example 20 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testFifoSchedulingPolicy.

@Test(timeout = 10000)
public void testFifoSchedulingPolicy() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 3, 1f, "fair")));
    plan.getPlan().setDefaultPoolPath("A");
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    // 2 running.
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A", null), conf, null), sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A", null), conf, null);
    assertEquals(0.5f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals(0.5f, sessionA2.getClusterFraction(), EPSILON);
    // Change the resource plan to use fifo policy.
    plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 3, 1f, "fifo")));
    plan.getPlan().setDefaultPoolPath("A");
    wm.updateResourcePlanAsync(plan).get();
    assertEquals(1f, sessionA1.getClusterFraction(), EPSILON);
    assertEquals(0f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA2.getPoolName());
    // Add another session.
    WmTezSession sessionA3 = (WmTezSession) wm.getSession(null, mappingInput("A", null), conf, null);
    assertEquals(0f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA3.getPoolName());
    // Make sure the allocation is transfered correctly on return.
    sessionA1.returnToSessionManager();
    assertEquals(1f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA3.getPoolName());
    // Make sure reuse changes the FIFO order of the session.
    WmTezSession sessionA4 = (WmTezSession) wm.getSession(sessionA2, mappingInput("A", null), conf, null);
    assertSame(sessionA2, sessionA4);
    assertEquals(1f, sessionA3.getClusterFraction(), EPSILON);
    assertEquals(0f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals("A", sessionA2.getPoolName());
    sessionA3.returnToSessionManager();
    assertEquals(1f, sessionA2.getClusterFraction(), EPSILON);
    sessionA2.returnToSessionManager();
}