Examples with WMFullResourcePlan org.apache.hadoop.hive.metastore.api.WMFullResourcePlan used on opensource projects

Example 21 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testClusterChange.

@Test(timeout = 10000)
public void testClusterChange() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 2, 1f)));
    plan.getPlan().setDefaultPoolPath("A");
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession session1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf), session2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    assertEquals(0.5, session1.getClusterFraction(), EPSILON);
    assertEquals(0.5, session2.getClusterFraction(), EPSILON);
    qam.assertWasCalledAndReset();
    // If cluster info changes, qam should be called with the same fractions.
    wm.notifyOfClusterStateChange();
    assertEquals(0.5, session1.getClusterFraction(), EPSILON);
    assertEquals(0.5, session2.getClusterFraction(), EPSILON);
    qam.assertWasCalledAndReset();
    session1.returnToSessionManager();
    session2.returnToSessionManager();
}

Example 22 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testDisableEnable.

@Test(timeout = 10000)
public void testDisableEnable() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 1f)));
    plan.getPlan().setDefaultPoolPath("A");
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    TezSessionPool<WmTezSession> tezAmPool = wm.getTezAmPool();
    // 1 running, 1 queued.
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A", null), conf, null);
    final AtomicReference<WmTezSession> sessionA2 = new AtomicReference<>();
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final CountDownLatch cdl1 = new CountDownLatch(1);
    Thread t1 = new Thread(new GetSessionRunnable(sessionA2, wm, error, conf, cdl1, "A"));
    waitForThreadToBlock(cdl1, t1);
    checkError(error);
    // Remove the resource plan - disable WM. All the queries die.
    wm.updateResourcePlanAsync(null).get();
    t1.join();
    assertNotNull(error.get());
    assertNull(sessionA2.get());
    assertKilledByWm(sessionA1);
    assertEquals(0, tezAmPool.getCurrentSize());
    // No-op for session killed by WM.
    sessionA1.returnToSessionManager();
    assertEquals(0, tezAmPool.getCurrentSize());
    try {
        TezSessionState r = wm.getSession(null, mappingInput("A", null), conf, null);
        fail("Expected an error but got " + r);
    } catch (WorkloadManager.NoPoolMappingException ex) {
    // Ignore, this particular error is expected.
    }
    // Apply the plan again - enable WM.
    wm.updateResourcePlanAsync(plan).get();
    sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A", null), conf, null);
    assertEquals("A", sessionA1.getPoolName());
    sessionA1.returnToSessionManager();
    assertEquals(1, tezAmPool.getCurrentSize());
}

Example 23 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testMappings.

@Test(timeout = 10000)
public void testMappings() throws Exception {
    HiveConf conf = createConf();
    conf.set(ConfVars.HIVE_SERVER2_WM_ALLOW_ANY_POOL_VIA_JDBC.varname, "false");
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("u0"), pool("g0"), pool("g1"), pool("u2"), pool("a0")));
    plan.setMappings(Lists.newArrayList(mapping("USER", "u0", "u0", 0), mapping("APPLICATION", "a0", "a0", 0), mapping("GROUP", "g0", "g0", 0), mapping("GROUP", "g1", "g1", 1), mapping("USER", "u2", "u2", 2)));
    WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    // Test various combinations.
    verifyMapping(wm, conf, mappingInput("u0", groups("zzz")), "u0");
    verifyMapping(wm, conf, new MappingInput("u0", null, null, "a0"), "u0");
    verifyMapping(wm, conf, new MappingInput("zzz", groups("g0"), null, "a0"), "a0");
    verifyMapping(wm, conf, mappingInput("zzz", groups("g1")), "g1");
    verifyMapping(wm, conf, mappingInput("u0", groups("g1")), "u0");
    // User takes precendence over groups unless ordered explicitly.
    verifyMapping(wm, conf, mappingInput("u0", groups("g0")), "u0");
    verifyMapping(wm, conf, mappingInput("u2", groups("g1")), "g1");
    verifyMapping(wm, conf, mappingInput("u2", groups("g0", "g1")), "g0");
    // Check explicit pool specifications - valid cases where priority is changed.
    verifyMapping(wm, conf, mappingInput("u0", groups("g1"), "g1"), "g1");
    verifyMapping(wm, conf, mappingInput("u2", groups("g1"), "u2"), "u2");
    verifyMapping(wm, conf, mappingInput("zzz", groups("g0", "g1"), "g1"), "g1");
    // Explicit pool specification - invalid - there's no mapping that matches.
    try {
        TezSessionState r = wm.getSession(null, mappingInput("u0", groups("g0", "g1"), "u2"), conf);
        fail("Expected failure, but got " + r);
    } catch (Exception ex) {
    // Expected.
    }
    // Now allow the users to specify any pools.
    conf.set(ConfVars.HIVE_SERVER2_WM_ALLOW_ANY_POOL_VIA_JDBC.varname, "true");
    wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    verifyMapping(wm, conf, mappingInput("u0", groups("g0", "g1"), "u2"), "u2");
    // The mapping that doesn't exist still shouldn't work.
    try {
        TezSessionState r = wm.getSession(null, mappingInput("u0", groups("g0", "g1"), "zzz"), conf);
        fail("Expected failure, but got " + r);
    } catch (Exception ex) {
    // Expected.
    }
}

Example 24 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class DDLTask method alterResourcePlan.

private int alterResourcePlan(Hive db, AlterResourcePlanDesc desc) throws HiveException {
    if (desc.shouldValidate()) {
        WMValidateResourcePlanResponse result = db.validateResourcePlan(desc.getResourcePlanName());
        try (DataOutputStream out = getOutputStream(desc.getResFile())) {
            formatter.showErrors(out, result);
        } catch (IOException e) {
            throw new HiveException(e);
        }
        ;
        return 0;
    }
    WMNullableResourcePlan resourcePlan = desc.getResourcePlan();
    final WorkloadManager wm = WorkloadManager.getInstance();
    final TezSessionPoolManager pm = TezSessionPoolManager.getInstance();
    boolean isActivate = false, isInTest = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST);
    if (resourcePlan.getStatus() != null) {
        isActivate = resourcePlan.getStatus() == WMResourcePlanStatus.ACTIVE;
    }
    WMFullResourcePlan appliedRp = db.alterResourcePlan(desc.getResourcePlanName(), resourcePlan, desc.isEnableActivate(), desc.isForceDeactivate(), desc.isReplace());
    boolean mustHaveAppliedChange = isActivate || desc.isForceDeactivate();
    if (!mustHaveAppliedChange && !desc.isReplace()) {
        // The modification cannot affect an active plan.
        return 0;
    }
    if (appliedRp == null && !mustHaveAppliedChange) {
        // Replacing an inactive plan.
        return 0;
    }
    if (wm == null && isInTest) {
        // Skip for tests if WM is not present.
        return 0;
    }
    if ((appliedRp == null) != desc.isForceDeactivate()) {
        throw new HiveException("Cannot get a resource plan to apply; or non-null plan on disable");
    // TODO: shut down HS2?
    }
    assert appliedRp == null || appliedRp.getPlan().getStatus() == WMResourcePlanStatus.ACTIVE;
    handleWorkloadManagementServiceChange(wm, pm, isActivate, appliedRp);
    return 0;
}

Example 25 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class HiveServer2 method createTestResourcePlan.

private WMFullResourcePlan createTestResourcePlan() {
    WMFullResourcePlan resourcePlan;
    WMPool pool = new WMPool("testDefault", "llap");
    pool.setAllocFraction(1f);
    pool.setQueryParallelism(1);
    resourcePlan = new WMFullResourcePlan(new WMResourcePlan("testDefault"), Lists.newArrayList(pool));
    resourcePlan.getPlan().setDefaultPoolPath("testDefault");
    return resourcePlan;
}