Examples with AggregatedPoolStats com.linkedin.pinot.transport.metrics.AggregatedPoolStats used on opensource projects

Example 1 with AggregatedPoolStats

use of com.linkedin.pinot.transport.metrics.AggregatedPoolStats in project pinot by linkedin.

the class KeyedPoolImplTest method testPoolImpl1.

@Test
public /**
   * Pool contains 5 inner pools with 5 resources as max capacity
   * Checkout all of them. Verify checked out is expected. Verify aggregated stats
   * Checkin all of them. At each step check stats
   * Checkout one from each inner pool and destroy them. Check stats
   * Shutdown. Ensure clean shutdown.
   * @throws Exception
   */
void testPoolImpl1() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 5;
    int numResourcesPerKey = 5;
    TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(5, 5, 1000 * 60 * 60L, 100, rm, timedExecutor, service, null);
    kPool.start();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    int c = 1;
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
            String resource = rFuture.getOne();
            Assert.assertEquals(resource, getResource(i, j));
            s.refresh();
            Assert.assertEquals(s.getCheckedOut(), c++);
        }
    }
    s = (AggregatedPoolStats) kPool.getStats();
    Assert.assertEquals(s.getTotalCreated(), numKeys * numResourcesPerKey);
    int checkedOut = c - 1;
    // checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            kPool.checkinObject(getKey(i), getResource(i, j));
            s.refresh();
            Assert.assertEquals(s.getCheckedOut(), --checkedOut);
        }
    }
    s = (AggregatedPoolStats) kPool.getStats();
    // checkout one from each and destroy them
    c = 1;
    int d = 1;
    for (int i = 0; i < numKeys; i++) {
        KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
        String resource = rFuture.getOne();
        Assert.assertEquals(resource, getResource(i, 0));
        CountDownLatch latch = new CountDownLatch(1);
        rm.setCountDownLatch(latch);
        kPool.destroyObject(getKey(i), resource);
        latch.await();
        Thread.sleep(1000);
        s.refresh();
        Assert.assertEquals(s.getCheckedOut(), 0);
        Assert.assertEquals(s.getTotalDestroyed(), d++);
    }
    Future<Map<String, NoneType>> f = kPool.shutdown();
    f.get();
    //Verify all objects are destroyed
    Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
    Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
    for (int i = 0; i < numKeys; i++) {
        List<String> r = destroyedMap.get(getKey(i));
        Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
        for (int j = 0; j < numResourcesPerKey; j++) {
            Assert.assertTrue(r.contains(getResource(i, j)));
        }
    }
}

Example 2 with AggregatedPoolStats

use of com.linkedin.pinot.transport.metrics.AggregatedPoolStats in project pinot by linkedin.

the class KeyedPoolImplTest method testTimeout.

@Test
public /**
   * IdleTimeout = 1sec
   * Pool => 5 keys. 1 resource per key ( 0 min, 5 max).
   *
   * 1. Checkout and checkin object to ensure they are created
   * 2. Wait for destroy latch to ensure the objects are deleted after they timeout
   * 3. Verify metrics
   * 4. Ensure shutdown succeeds
   *
   * @throws Exception
   */
void testTimeout() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 5;
    int numResourcesPerKey = 1;
    TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
    // Idle Timeout 1 second
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 5, 1000L, 100, rm, timedExecutor, service, null);
    // Create a countdown latch that waits for all resources to be deleted
    CountDownLatch latch = new CountDownLatch(numKeys * numResourcesPerKey);
    rm.setCountDownLatch(latch);
    kPool.start();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    // checkout and checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
            String resource = rFuture.getOne();
        }
    }
    // checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            kPool.checkinObject(getKey(i), getResource(i, j));
        }
    }
    //Wait for all to be destroyed
    latch.await();
    s.refresh();
    Assert.assertEquals(s.getTotalTimedOut(), 5);
    //Verify all objects are destroyed
    Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
    Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
    for (int i = 0; i < numKeys; i++) {
        List<String> r = destroyedMap.get(getKey(i));
        Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
        for (int j = 0; j < numResourcesPerKey; j++) {
            Assert.assertTrue(r.contains(getResource(i, j)));
        }
    }
    // Proper shutdown
    Future<Map<String, NoneType>> f = kPool.shutdown();
    f.get();
}

Example 3 with AggregatedPoolStats

use of com.linkedin.pinot.transport.metrics.AggregatedPoolStats in project pinot by linkedin.

the class KeyedPoolImplTest method testDestroyError.

@Test
public void testDestroyError() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 1;
    int numResourcesPerKey = 1;
    Map<String, List<String>> resources = buildCreateMap(numKeys, numResourcesPerKey);
    TestResourceManager rm = new TestResourceManager(resources, null, resources, null);
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 5, 1000L, 1000 * 60 * 60, rm, timedExecutor, service, null);
    AsyncResponseFuture<String, String> f = (AsyncResponseFuture<String, String>) kPool.checkoutObject(getKey(0));
    String r = f.getOne();
    Assert.assertTrue(f.isDone());
    Assert.assertNull(f.getError());
    // Create a countdown latch that waits for the attempt to delete the resource
    CountDownLatch latch = new CountDownLatch(1);
    rm.setCountDownLatch(latch);
    kPool.destroyObject(getKey(0), r);
    latch.await();
    // shutdown
    kPool.shutdown().get();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    s.refresh();
    Assert.assertEquals(s.getTotalDestroyErrors(), 1);
}

Example 4 with AggregatedPoolStats

use of com.linkedin.pinot.transport.metrics.AggregatedPoolStats in project pinot by linkedin.

the class KeyedPoolImplTest method testCreateError.

@Test
public void testCreateError() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 1;
    int numResourcesPerKey = 1;
    Map<String, List<String>> resources = buildCreateMap(numKeys, numResourcesPerKey);
    TestResourceManager rm = new TestResourceManager(resources, resources, null, null);
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 1, 1000L, 1000 * 60 * 60, rm, timedExecutor, service, null);
    AsyncResponseFuture<String, String> f = (AsyncResponseFuture<String, String>) kPool.checkoutObject(getKey(0));
    Assert.assertTrue(f.isDone());
    Assert.assertNull(f.get());
    Assert.assertNotNull(f.getError());
    kPool.shutdown().get();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    s.refresh();
    Assert.assertEquals(s.getTotalCreateErrors(), 1);
}

Example 5 with AggregatedPoolStats

use of com.linkedin.pinot.transport.metrics.AggregatedPoolStats in project pinot by linkedin.

the class KeyedPoolImplTest method testShutdown.

@Test
public /**
   * Pool contains 5 inner pools with 5 resources as max capacity
   * First checkout and checkin all resources
   * Checkout one from each inner pool.
   * Shutdown now. This should not complete.
   * Destroy the checked out objects. Check stats
   * shutdown should have happened
   * @throws Exception
   */
void testShutdown() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 5;
    int numResourcesPerKey = 5;
    TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(5, 5, 1000 * 60 * 60L, 100, rm, timedExecutor, service, null);
    kPool.start();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    int c = 1;
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
            String resource = rFuture.getOne();
            Assert.assertEquals(resource, getResource(i, j));
            s.refresh();
            Assert.assertEquals(s.getCheckedOut(), c++);
        }
    }
    s = (AggregatedPoolStats) kPool.getStats();
    Assert.assertEquals(s.getTotalCreated(), numKeys * numResourcesPerKey);
    int checkedOut = c - 1;
    // checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            kPool.checkinObject(getKey(i), getResource(i, j));
            s.refresh();
            Assert.assertEquals(s.getCheckedOut(), --checkedOut);
        }
    }
    // Check out 1 object for each key
    c = 1;
    for (int i = 0; i < numKeys; i++) {
        KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
        String resource = rFuture.getOne();
        Assert.assertEquals(resource, getResource(i, 0));
        s.refresh();
        Assert.assertEquals(s.getCheckedOut(), c);
        c++;
    }
    Assert.assertEquals(s.getPoolSize(), numKeys * numResourcesPerKey);
    Assert.assertEquals(s.getIdleCount(), (numKeys * numResourcesPerKey) - 5);
    // SHutdown but it should not be done.
    Future<Map<String, NoneType>> f = kPool.shutdown();
    FutureReader<Map<String, NoneType>> reader = new FutureReader<Map<String, NoneType>>(f);
    reader.start();
    reader.getBeginLatch().await();
    Assert.assertTrue(reader.isStarted());
    Assert.assertFalse(reader.isDone());
    //none are destroyed
    Assert.assertEquals(rm.getDestroyedMap().keySet().size(), 0);
    // Now destroy some and checkin others
    int d = 0;
    for (int i = 0; i < numKeys; i++) {
        if ((i % 2) == 0) {
            kPool.destroyObject(getKey(i), getResource(i, 0));
            s.refresh();
            Assert.assertEquals(s.getTotalDestroyed(), ++d);
        } else {
            kPool.checkinObject(getKey(i), getResource(i, 0));
        }
    }
    s.refresh();
    Assert.assertEquals(s.getTotalDestroyed(), 3);
    // Now shutdown should complete
    f.get();
    reader.getEndLatch().await();
    Assert.assertTrue(reader.isDone());
    // Do one more shutdown call
    Future<Map<String, NoneType>> f2 = kPool.shutdown();
    f2.get();
    //Verify all objects are destroyed
    Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
    Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
    for (int i = 0; i < numKeys; i++) {
        List<String> r = destroyedMap.get(getKey(i));
        Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
        for (int j = 0; j < numResourcesPerKey; j++) {
            Assert.assertTrue(r.contains(getResource(i, j)));
        }
    }
}