use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class KeyedPoolImplTest method testPoolImpl1.
@Test
public /**
* Pool contains 5 inner pools with 5 resources as max capacity
* Checkout all of them. Verify checked out is expected. Verify aggregated stats
* Checkin all of them. At each step check stats
* Checkout one from each inner pool and destroy them. Check stats
* Shutdown. Ensure clean shutdown.
* @throws Exception
*/
void testPoolImpl1() throws Exception {
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService service = MoreExecutors.sameThreadExecutor();
int numKeys = 5;
int numResourcesPerKey = 5;
TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(5, 5, 1000 * 60 * 60L, 100, rm, timedExecutor, service, null);
kPool.start();
AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
int c = 1;
for (int j = 0; j < numResourcesPerKey; j++) {
for (int i = 0; i < numKeys; i++) {
KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
String resource = rFuture.getOne();
Assert.assertEquals(resource, getResource(i, j));
s.refresh();
Assert.assertEquals(s.getCheckedOut(), c++);
}
}
s = (AggregatedPoolStats) kPool.getStats();
Assert.assertEquals(s.getTotalCreated(), numKeys * numResourcesPerKey);
int checkedOut = c - 1;
// checkin back all
for (int j = 0; j < numResourcesPerKey; j++) {
for (int i = 0; i < numKeys; i++) {
kPool.checkinObject(getKey(i), getResource(i, j));
s.refresh();
Assert.assertEquals(s.getCheckedOut(), --checkedOut);
}
}
s = (AggregatedPoolStats) kPool.getStats();
// checkout one from each and destroy them
c = 1;
int d = 1;
for (int i = 0; i < numKeys; i++) {
KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
String resource = rFuture.getOne();
Assert.assertEquals(resource, getResource(i, 0));
CountDownLatch latch = new CountDownLatch(1);
rm.setCountDownLatch(latch);
kPool.destroyObject(getKey(i), resource);
latch.await();
Thread.sleep(1000);
s.refresh();
Assert.assertEquals(s.getCheckedOut(), 0);
Assert.assertEquals(s.getTotalDestroyed(), d++);
}
Future<Map<String, NoneType>> f = kPool.shutdown();
f.get();
//Verify all objects are destroyed
Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
for (int i = 0; i < numKeys; i++) {
List<String> r = destroyedMap.get(getKey(i));
Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
for (int j = 0; j < numResourcesPerKey; j++) {
Assert.assertTrue(r.contains(getResource(i, j)));
}
}
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class KeyedPoolImplTest method testInvalidCheckinDestroy.
@Test
public void testInvalidCheckinDestroy() throws Exception {
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
int numKeys = 1;
int numResourcesPerKey = 1;
Map<String, List<String>> resources = buildCreateMap(numKeys, numResourcesPerKey);
TestResourceManager rm = new TestResourceManager(resources, null, null, null);
KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 1, 1000L, 1000 * 60 * 60, rm, timedExecutor, service, null);
kPool.start();
AsyncResponseFuture<String, String> f = (AsyncResponseFuture<String, String>) kPool.checkoutObject(getKey(0));
String s1 = f.getOne();
// checkin with invalid key
boolean isException = false;
try {
kPool.checkinObject(getKey(1), s1);
} catch (IllegalStateException e) {
isException = true;
}
Assert.assertTrue(isException);
// destroy with invalid key
isException = false;
try {
kPool.destroyObject(getKey(1), s1);
} catch (IllegalStateException e) {
isException = true;
}
Assert.assertTrue(isException);
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class KeyedPoolImplTest method testCancelAfterCheckingOut.
@Test
public void testCancelAfterCheckingOut() throws Exception {
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
int numKeys = 1;
int numResourcesPerKey = 1;
Map<String, List<String>> resources = buildCreateMap(numKeys, numResourcesPerKey);
BlockingTestResourceManager rm = new BlockingTestResourceManager(resources, null, null, null);
KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 1, 1000L, 1000 * 60 * 60, rm, timedExecutor, service, null);
kPool.start();
AsyncResponseFuture<String, String> f = (AsyncResponseFuture<String, String>) kPool.checkoutObject(getKey(0));
boolean isTimedout = false;
try {
f.get(2, TimeUnit.SECONDS);
} catch (TimeoutException e) {
isTimedout = true;
}
Assert.assertTrue(isTimedout);
boolean cancelled = f.cancel(false);
Assert.assertTrue(cancelled);
Assert.assertTrue(f.isCancelled());
Assert.assertTrue(f.isDone());
rm.getCreateBlockLatch().countDown();
kPool.shutdown().get();
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class KeyedPoolImplTest method testTimeout.
@Test
public /**
* IdleTimeout = 1sec
* Pool => 5 keys. 1 resource per key ( 0 min, 5 max).
*
* 1. Checkout and checkin object to ensure they are created
* 2. Wait for destroy latch to ensure the objects are deleted after they timeout
* 3. Verify metrics
* 4. Ensure shutdown succeeds
*
* @throws Exception
*/
void testTimeout() throws Exception {
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService service = MoreExecutors.sameThreadExecutor();
int numKeys = 5;
int numResourcesPerKey = 1;
TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
// Idle Timeout 1 second
KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 5, 1000L, 100, rm, timedExecutor, service, null);
// Create a countdown latch that waits for all resources to be deleted
CountDownLatch latch = new CountDownLatch(numKeys * numResourcesPerKey);
rm.setCountDownLatch(latch);
kPool.start();
AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
// checkout and checkin back all
for (int j = 0; j < numResourcesPerKey; j++) {
for (int i = 0; i < numKeys; i++) {
KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
String resource = rFuture.getOne();
}
}
// checkin back all
for (int j = 0; j < numResourcesPerKey; j++) {
for (int i = 0; i < numKeys; i++) {
kPool.checkinObject(getKey(i), getResource(i, j));
}
}
//Wait for all to be destroyed
latch.await();
s.refresh();
Assert.assertEquals(s.getTotalTimedOut(), 5);
//Verify all objects are destroyed
Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
for (int i = 0; i < numKeys; i++) {
List<String> r = destroyedMap.get(getKey(i));
Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
for (int j = 0; j < numResourcesPerKey; j++) {
Assert.assertTrue(r.contains(getResource(i, j)));
}
}
// Proper shutdown
Future<Map<String, NoneType>> f = kPool.shutdown();
f.get();
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class ScatterGatherTest method testMultipleServerTimeout.
@Test
public void testMultipleServerTimeout() throws Exception {
MetricsRegistry registry = new MetricsRegistry();
// Server start
int serverPort1 = 7081;
int serverPort2 = 7082;
int serverPort3 = 7083;
// Timeout server
int serverPort4 = 7084;
NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 7000, false), null);
Thread t1 = new Thread(server1);
Thread t2 = new Thread(server2);
Thread t3 = new Thread(server3);
Thread t4 = new Thread(server4);
t1.start();
t2.start();
t3.start();
t4.start();
//Client setup
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService service = new ThreadPoolExecutor(5, 5, 5, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, service, registry);
rm.setPool(pool);
SegmentIdSet pg1 = new SegmentIdSet();
pg1.addSegment(new SegmentId("0"));
SegmentIdSet pg2 = new SegmentIdSet();
pg2.addSegment(new SegmentId("1"));
SegmentIdSet pg3 = new SegmentIdSet();
pg3.addSegment(new SegmentId("2"));
SegmentIdSet pg4 = new SegmentIdSet();
pg4.addSegment(new SegmentId("3"));
ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
pgMap.put(serverInstance1, pg1);
pgMap.put(serverInstance2, pg2);
pgMap.put(serverInstance3, pg3);
pgMap.put(serverInstance4, pg4);
String request1 = "request_0";
String request2 = "request_1";
String request3 = "request_2";
String request4 = "request_3";
Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
pgMapStr.put(pg1, request1);
pgMapStr.put(pg2, request2);
pgMapStr.put(pg3, request3);
pgMapStr.put(pg4, request4);
ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
Map<ServerInstance, ByteBuf> v = fut.get();
//Only 3 servers return value.
Assert.assertEquals(v.size(), 3);
ByteBuf b = v.get(serverInstance1);
byte[] b2 = new byte[b.readableBytes()];
b.readBytes(b2);
String response = new String(b2);
Assert.assertEquals(response, "response_0_0");
b = v.get(serverInstance2);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_1_0");
b = v.get(serverInstance3);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_2_0");
//No response from 4th server
Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
Map<ServerInstance, Throwable> errorMap = fut.getError();
Assert.assertEquals(errorMap.size(), 1, "One error");
Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
Thread.sleep(3000);
pool.getStats().refresh();
Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
pool.shutdown();
service.shutdown();
eventLoopGroup.shutdownGracefully();
server1.shutdownGracefully();
server2.shutdownGracefully();
server3.shutdownGracefully();
server4.shutdownGracefully();
}
Aggregations