Examples with WMFullResourcePlan org.apache.hadoop.hive.metastore.api.WMFullResourcePlan used on opensource projects

Example 6 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testAsyncSessionInitFailures.

@Test(timeout = 10000)
public void testAsyncSessionInitFailures() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 1.0f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A")));
    final WorkloadManagerForTest wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    // Make sure session init gets stuck in init.
    TezSessionPool<WmTezSession> pool = wm.getTezAmPool();
    SampleTezSessionState theOnlySession = (SampleTezSessionState) pool.getSession();
    SettableFuture<Boolean> blockedWait = SettableFuture.create();
    theOnlySession.setWaitForAmRegistryFuture(blockedWait);
    pool.returnSession(theOnlySession);
    assertEquals(1, pool.getCurrentSize());
    final AtomicReference<WmTezSession> sessionA = new AtomicReference<>();
    final AtomicReference<Throwable> error = new AtomicReference<>();
    CountDownLatch cdl = new CountDownLatch(1);
    Thread t1 = new Thread(new GetSessionRunnable(sessionA, wm, error, conf, cdl, "A"));
    waitForThreadToBlock(cdl, t1);
    checkError(error);
    wm.addTestEvent().get();
    // The session is taken out of the pool, but is waiting for registration.
    assertEquals(0, pool.getCurrentSize());
    // Change the resource plan, so that the session gets killed.
    plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("B", 1, 1.0f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "B")));
    wm.updateResourcePlanAsync(plan);
    wm.addTestEvent().get();
    // Meanwhile, the init succeeds!
    blockedWait.set(true);
    t1.join();
    try {
        sessionA.get();
        fail("Expected an error but got " + sessionA.get());
    } catch (Throwable t) {
    // Expected.
    }
    try {
        // The get-session call should also fail.
        checkError(error);
        fail("Expected an error");
    } catch (Exception ex) {
    // Expected.
    }
    error.set(null);
    theOnlySession = validatePoolAfterCleanup(theOnlySession, conf, wm, pool, "B");
    // Initialization fails with retry, no resource plan change.
    SettableFuture<Boolean> failedWait = SettableFuture.create();
    failedWait.setException(new Exception("foo"));
    theOnlySession.setWaitForAmRegistryFuture(failedWait);
    TezSessionState retriedSession = wm.getSession(null, mappingInput("A"), conf);
    assertNotNull(retriedSession);
    // Should have been replaced.
    assertNotSame(theOnlySession, retriedSession);
    retriedSession.returnToSessionManager();
    theOnlySession = (SampleTezSessionState) retriedSession;
    // Initialization fails and so does the retry, no resource plan change.
    theOnlySession.setWaitForAmRegistryFuture(failedWait);
    // Fail the retry.
    wm.setNextWaitForAmRegistryFuture(failedWait);
    try {
        TezSessionState r = wm.getSession(null, mappingInput("A"), conf);
        fail("Expected an error but got " + r);
    } catch (Exception ex) {
    // Expected.
    }
    theOnlySession = validatePoolAfterCleanup(theOnlySession, conf, wm, pool, "B");
    // Init fails, but the session is also killed by WM before that.
    failedWait = SettableFuture.create();
    theOnlySession.setWaitForAmRegistryFuture(failedWait);
    // Fail the retry.
    wm.setNextWaitForAmRegistryFuture(failedWait);
    sessionA.set(null);
    cdl = new CountDownLatch(1);
    t1 = new Thread(new GetSessionRunnable(sessionA, wm, error, conf, cdl, "A"));
    waitForThreadToBlock(cdl, t1);
    wm.addTestEvent().get();
    // The session is taken out of the pool, but is waiting for registration.
    assertEquals(0, pool.getCurrentSize());
    plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 1.0f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A")));
    wm.updateResourcePlanAsync(plan);
    wm.addTestEvent().get();
    // Meanwhile, the init fails.
    failedWait.setException(new Exception("moo"));
    t1.join();
    try {
        sessionA.get();
        fail("Expected an error but got " + sessionA.get());
    } catch (Throwable t) {
    // Expected.
    }
    try {
        // The get-session call should also fail.
        checkError(error);
        fail("Expected an error");
    } catch (Exception ex) {
    // Expected.
    }
    validatePoolAfterCleanup(theOnlySession, conf, wm, pool, "A");
}

Example 7 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testQueueing.

@Test(timeout = 10000)
public void testQueueing() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 2, 0.5f), pool("B", 2, 0.5f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf), sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf), sessionB1 = (WmTezSession) wm.getSession(null, mappingInput("B"), conf);
    final AtomicReference<WmTezSession> sessionA3 = new AtomicReference<>(), sessionA4 = new AtomicReference<>();
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final CountDownLatch cdl = new CountDownLatch(1);
    Thread t1 = new Thread(new GetSessionRunnable(sessionA3, wm, error, conf, cdl, "A")), t2 = new Thread(new GetSessionRunnable(sessionA4, wm, error, conf, null, "A"));
    waitForThreadToBlock(cdl, t1);
    t2.start();
    assertNull(sessionA3.get());
    assertNull(sessionA4.get());
    checkError(error);
    // While threads are blocked on A, we should still be able to get and return a B session.
    WmTezSession sessionB2 = (WmTezSession) wm.getSession(null, mappingInput("B"), conf);
    sessionB1.returnToSessionManager();
    sessionB2.returnToSessionManager();
    assertNull(sessionA3.get());
    assertNull(sessionA4.get());
    checkError(error);
    // Now release a single session from A.
    sessionA1.returnToSessionManager();
    t1.join();
    checkError(error);
    assertNotNull(sessionA3.get());
    assertNull(sessionA4.get());
    sessionA3.get().returnToSessionManager();
    t2.join();
    checkError(error);
    assertNotNull(sessionA4.get());
    sessionA4.get().returnToSessionManager();
    sessionA2.returnToSessionManager();
}

Example 8 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testClusterFractions.

@Test(timeout = 10000)
public void testClusterFractions() throws Exception {
    HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("r1", 1, 0.6f), pool("r2", 1, 0.4f), pool("r1.p1", 1, 0.5f), pool("r1.p2", 2, 0.3f)));
    plan.setMappings(Lists.newArrayList(mapping("p1", "r1.p1"), mapping("p2", "r1.p2"), mapping("r1", "r1"), mapping("r2", "r2")));
    WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    assertEquals(5, wm.getNumSessions());
    // Get all the 5 sessions; validate cluster fractions.
    WmTezSession session05of06 = (WmTezSession) wm.getSession(null, mappingInput("p1"), conf);
    assertEquals(0.3, session05of06.getClusterFraction(), EPSILON);
    WmTezSession session03of06 = (WmTezSession) wm.getSession(null, mappingInput("p2"), conf);
    assertEquals(0.18, session03of06.getClusterFraction(), EPSILON);
    WmTezSession session03of06_2 = (WmTezSession) wm.getSession(null, mappingInput("p2"), conf);
    assertEquals(0.09, session03of06.getClusterFraction(), EPSILON);
    assertEquals(0.09, session03of06_2.getClusterFraction(), EPSILON);
    WmTezSession session02of06 = (WmTezSession) wm.getSession(null, mappingInput("r1"), conf);
    assertEquals(0.12, session02of06.getClusterFraction(), EPSILON);
    WmTezSession session04 = (WmTezSession) wm.getSession(null, mappingInput("r2"), conf);
    assertEquals(0.4, session04.getClusterFraction(), EPSILON);
    session05of06.returnToSessionManager();
    session03of06.returnToSessionManager();
    session03of06_2.returnToSessionManager();
    session02of06.returnToSessionManager();
    session04.returnToSessionManager();
}

Example 9 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestWorkloadManager method testMoveSessionsMultiPool.

@Test(timeout = 10000)
public void testMoveSessionsMultiPool() throws Exception {
    final HiveConf conf = createConf();
    MockQam qam = new MockQam();
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(pool("A", 1, 0.4f), pool("B", 1, 0.4f), pool("B.x", 1, 0.2f), pool("B.y", 1, 0.8f), pool("C", 1, 0.2f)));
    plan.setMappings(Lists.newArrayList(mapping("A", "A"), mapping("B", "B"), mapping("C", "C")));
    final WorkloadManager wm = new WorkloadManagerForTest("test", conf, qam, plan);
    wm.start();
    WmTezSession sessionA1 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0, B.x: 0, B.y: 0, C: 0]
    Map<String, SessionTriggerProvider> allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.4f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA1));
    assertEquals("A", sessionA1.getPoolName());
    // [A: 0, B: 1, B.x: 0, B.y: 0, C: 0]
    Future<Boolean> future = wm.applyMoveSessionAsync(sessionA1, "B.y");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.32f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("B.y").getSessions().contains(sessionA1));
    assertEquals("B.y", sessionA1.getPoolName());
    // [A: 0, B: 0, B.x: 0, B.y: 0, C: 1]
    future = wm.applyMoveSessionAsync(sessionA1, "C");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(1, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.2f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("C").getSessions().contains(sessionA1));
    assertEquals("C", sessionA1.getPoolName());
    // [A: 0, B: 0, B.x: 1, B.y: 0, C: 0]
    future = wm.applyMoveSessionAsync(sessionA1, "B.x");
    assertNotNull(future.get());
    assertTrue(future.get());
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.08f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("B.x").getSessions().contains(sessionA1));
    assertEquals("B.x", sessionA1.getPoolName());
    WmTezSession sessionA2 = (WmTezSession) wm.getSession(null, mappingInput("A"), conf);
    // [A: 1, B: 0, B.x: 1, B.y: 0, C: 0]
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(1, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.4f, sessionA2.getClusterFraction(), EPSILON);
    assertEquals(0.08f, sessionA1.getClusterFraction(), EPSILON);
    assertTrue(allSessionProviders.get("A").getSessions().contains(sessionA2));
    assertTrue(allSessionProviders.get("B.x").getSessions().contains(sessionA1));
    assertEquals("A", sessionA2.getPoolName());
    assertEquals("B.x", sessionA1.getPoolName());
    // A is maxed out on capacity, so this move should fail the session
    // [A: 1, B: 0, B.x: 0, B.y: 0, C: 0]
    future = wm.applyMoveSessionAsync(sessionA1, "A");
    assertNotNull(future.get());
    assertFalse(future.get());
    wm.addTestEvent().get();
    while (sessionA1.isOpen()) {
        Thread.sleep(100);
    }
    assertNull(sessionA1.getPoolName());
    assertEquals("Destination pool A is full. Killing query.", sessionA1.getReasonForKill());
    assertEquals(1, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    // return a loaned session goes back to tez am pool
    // [A: 0, B: 0, B.x: 0, B.y: 0, C: 0]
    wm.returnAfterUse(sessionA2);
    wm.addTestEvent().get();
    allSessionProviders = wm.getAllSessionTriggerProviders();
    assertEquals(0, allSessionProviders.get("A").getSessions().size());
    assertEquals(0, allSessionProviders.get("B").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.x").getSessions().size());
    assertEquals(0, allSessionProviders.get("B.y").getSessions().size());
    assertEquals(0, allSessionProviders.get("C").getSessions().size());
    assertEquals(0.0f, sessionA1.getClusterFraction(), EPSILON);
    assertFalse(allSessionProviders.get("A").getSessions().contains(sessionA1));
}

Example 10 with WMFullResourcePlan

use of org.apache.hadoop.hive.metastore.api.WMFullResourcePlan in project hive by apache.

the class TestJsonRPFormatter method testJsonEmptyRPFormatter.

@Test
public void testJsonEmptyRPFormatter() throws Exception {
    WMFullResourcePlan fullRp = createRP("test_rp_1", null, null);
    formatter.showFullResourcePlan(out, fullRp);
    out.flush();
    ObjectMapper objectMapper = new ObjectMapper();
    JsonNode jsonTree = objectMapper.readTree(bos.toByteArray());
    assertNotNull(jsonTree);
    assertTrue(jsonTree.isObject());
    assertEquals("test_rp_1", jsonTree.get("name").asText());
    assertTrue(jsonTree.get("parallelism").isNull());
    assertTrue(jsonTree.get("defaultPool").isNull());
    assertTrue(jsonTree.get("pools").isArray());
    assertEquals(0, jsonTree.get("pools").size());
}