Examples with ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor used on opensource projects

Example 91 with ScheduledThreadPoolExecutor

use of java.util.concurrent.ScheduledThreadPoolExecutor in project hbase by apache.

the class TestCanaryTool method testBasicZookeeperCanaryWorks.

@Test
public void testBasicZookeeperCanaryWorks() throws Exception {
    Integer port = Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
    testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, "localhost:" + port + "/hbase");
    ExecutorService executor = new ScheduledThreadPoolExecutor(2);
    Canary.ZookeeperStdOutSink sink = spy(new Canary.ZookeeperStdOutSink());
    Canary canary = new Canary(executor, sink);
    String[] args = { "-t", "10000", "-zookeeper" };
    ToolRunner.run(testingUtility.getConfiguration(), canary, args);
    String baseZnode = testingUtility.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
    verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
}

Example 92 with ScheduledThreadPoolExecutor

use of java.util.concurrent.ScheduledThreadPoolExecutor in project hbase by apache.

the class TestHBaseFsckTwoRS method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
    conf.setInt("hbase.regionserver.handler.count", 2);
    conf.setInt("hbase.regionserver.metahandler.count", 30);
    conf.setInt("hbase.htable.threads.max", POOL_SIZE);
    conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
    conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
    TEST_UTIL.startMiniCluster(2);
    tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("testhbck"));
    hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
    AssignmentManager assignmentManager = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
    regionStates = assignmentManager.getRegionStates();
    connection = (ClusterConnection) TEST_UTIL.getConnection();
    admin = connection.getAdmin();
    admin.setBalancerRunning(false, true);
    TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
    TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME);
}

Example 93 with ScheduledThreadPoolExecutor

use of java.util.concurrent.ScheduledThreadPoolExecutor in project pinot by linkedin.

the class ScatterGatherTest method testSingleServer.

@Test
public void testSingleServer() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort = 7071;
    NettyTCPServer server1 = new NettyTCPServer(serverPort, new TestRequestHandlerFactory(0, 1), null);
    Thread t1 = new Thread(server1);
    t1.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    SegmentIdSet pg = new SegmentIdSet();
    pg.addSegment(new SegmentId("0"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort);
    List<ServerInstance> instances = new ArrayList<ServerInstance>();
    instances.add(serverInstance1);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg);
    String request = "request_0";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg, request);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    Assert.assertEquals(v.size(), 1);
    server1.shutdownGracefully();
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
}

Example 94 with ScheduledThreadPoolExecutor

use of java.util.concurrent.ScheduledThreadPoolExecutor in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerError.

@Test
public void testMultipleServerError() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7091;
    int serverPort2 = 7092;
    int serverPort3 = 7093;
    // error server
    int serverPort4 = 7094;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 1000, true), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    final BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    //Only 3 servers return value.
    Assert.assertEquals(v.size(), 3);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    //No  response from 4th server
    Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
    Map<ServerInstance, Throwable> errorMap = fut.getError();
    Assert.assertEquals(errorMap.size(), 1, "One error");
    Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
    //    System.out.println("Error is :" + errorMap.get(serverInstance4));
    Thread.sleep(3000);
    //    System.out.println("Pool Stats :" + pool.getStats());
    pool.getStats().refresh();
    Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
}

Example 95 with ScheduledThreadPoolExecutor

use of java.util.concurrent.ScheduledThreadPoolExecutor in project pinot by linkedin.

the class NettySingleConnectionIntegrationTest method testValidatePool.

/*
   * WARNING: This test has potential failures due to timing.
   */
@Test
public void testValidatePool() throws Exception {
    NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
    Timer timer = new HashedWheelTimer();
    MyServer server = new MyServer();
    Thread.sleep(1000);
    // used as a key to pool. Can be anything.
    final String serverName = "SomeServer";
    ServerInstance serverInstance = server.getServerInstance();
    MetricsRegistry metricsRegistry = new MetricsRegistry();
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    PooledNettyClientResourceManager resourceManager = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), metric);
    ExecutorService executorService = Executors.newCachedThreadPool();
    ScheduledExecutorService timeoutExecutor = new ScheduledThreadPoolExecutor(5);
    AsyncPoolResourceManagerAdapter<ServerInstance, NettyClientConnection> rmAdapter = new AsyncPoolResourceManagerAdapter<ServerInstance, NettyClientConnection>(serverInstance, resourceManager, executorService, metricsRegistry);
    AsyncPool pool = new AsyncPoolImpl<NettyClientConnection>(serverName, rmAdapter, /*maxSize=*/
    5, /*idleTimeoutMs=*/
    100000, timeoutExecutor, executorService, /*maxWaiters=*/
    10, AsyncPoolImpl.Strategy.LRU, /*minSize=*/
    2, metricsRegistry);
    pool.start();
    Callback<NoneType> callback;
    callback = new Callback<NoneType>() {

        @Override
        public void onSuccess(NoneType arg0) {
        }

        @Override
        public void onError(Throwable arg0) {
            Assert.fail("Shutdown error");
        }
    };
    boolean serverShutdown = false;
    try {
        PoolStats stats;
        /* Validate with no connection in pool */
        // Give the pool enough time to create connections (in this case, 2 connections minSize)
        Thread.sleep(3000);
        //      System.out.println("Validating with no used objects in the pool");
        pool.validate(false);
        // System.out.println(stats);
        stats = pool.getStats();
        Assert.assertEquals(2, stats.getPoolSize());
        Assert.assertEquals(0, stats.getTotalBadDestroyed());
        /* checkout one connection, it should not destroy anything */
        AsyncResponseFuture<ServerInstance, NettyClientConnection> future = new AsyncResponseFuture<ServerInstance, NettyClientConnection>(serverInstance, "Future for " + serverName);
        Cancellable cancellable = pool.get(future);
        future.setCancellable(cancellable);
        NettyClientConnection conn = future.getOne();
        // System.out.println(stats);
        stats = pool.getStats();
        //      System.out.println("Validating with one used object in the pool");
        pool.validate(false);
        Assert.assertEquals(2, stats.getPoolSize());
        Assert.assertEquals(0, stats.getTotalBadDestroyed());
        Assert.assertEquals(1, stats.getCheckedOut());
        // Now stop the server, so that the checked out connection is invalidated.
        server.shutdown();
        serverShutdown = true;
        ;
        // Wait for the client channel to be closed.
        Thread.sleep(2000);
        pool.validate(false);
        // Wait for the callback into AsyncPoolImpl after the destroy thread completes destroying the connection
        Thread.sleep(5000);
        //      System.out.println("Validating with one used object in the pool, after server shutdown");
        // System.out.println(stats);
        stats = pool.getStats();
        Assert.assertEquals(2, stats.getPoolSize());
        Assert.assertEquals(1, stats.getTotalBadDestroyed());
        Assert.assertEquals(1, stats.getCheckedOut());
    } finally {
        server.shutdown();
        pool.shutdown(callback);
        executorService.shutdown();
        timeoutExecutor.shutdown();
    }
}