Examples with ScheduledExecutorService java.util.concurrent.ScheduledExecutorService used on opensource projects

Example 11 with ScheduledExecutorService

use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.

the class KeyedPoolImplTest method testTimeout.

@Test
public /**
   * IdleTimeout = 1sec
   * Pool => 5 keys. 1 resource per key ( 0 min, 5 max).
   *
   * 1. Checkout and checkin object to ensure they are created
   * 2. Wait for destroy latch to ensure the objects are deleted after they timeout
   * 3. Verify metrics
   * 4. Ensure shutdown succeeds
   *
   * @throws Exception
   */
void testTimeout() throws Exception {
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = MoreExecutors.sameThreadExecutor();
    int numKeys = 5;
    int numResourcesPerKey = 1;
    TestResourceManager rm = new TestResourceManager(buildCreateMap(numKeys, numResourcesPerKey), null, null, null);
    // Idle Timeout 1 second
    KeyedPool<String, String> kPool = new KeyedPoolImpl<String, String>(0, 5, 1000L, 100, rm, timedExecutor, service, null);
    // Create a countdown latch that waits for all resources to be deleted
    CountDownLatch latch = new CountDownLatch(numKeys * numResourcesPerKey);
    rm.setCountDownLatch(latch);
    kPool.start();
    AggregatedPoolStats s = (AggregatedPoolStats) kPool.getStats();
    // checkout and checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            KeyedFuture<String, String> rFuture = kPool.checkoutObject(getKey(i));
            String resource = rFuture.getOne();
        }
    }
    // checkin back all
    for (int j = 0; j < numResourcesPerKey; j++) {
        for (int i = 0; i < numKeys; i++) {
            kPool.checkinObject(getKey(i), getResource(i, j));
        }
    }
    //Wait for all to be destroyed
    latch.await();
    s.refresh();
    Assert.assertEquals(s.getTotalTimedOut(), 5);
    //Verify all objects are destroyed
    Map<String, List<String>> destroyedMap = rm.getDestroyedMap();
    Assert.assertEquals(destroyedMap.keySet().size(), numKeys);
    for (int i = 0; i < numKeys; i++) {
        List<String> r = destroyedMap.get(getKey(i));
        Assert.assertEquals(r.size(), numResourcesPerKey, "Resource for Key (" + getKey(i) + ")");
        for (int j = 0; j < numResourcesPerKey; j++) {
            Assert.assertTrue(r.contains(getResource(i, j)));
        }
    }
    // Proper shutdown
    Future<Map<String, NoneType>> f = kPool.shutdown();
    f.get();
}

Example 12 with ScheduledExecutorService

use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerTimeout.

@Test
public void testMultipleServerTimeout() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7081;
    int serverPort2 = 7082;
    int serverPort3 = 7083;
    // Timeout server
    int serverPort4 = 7084;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 7000, false), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = new ThreadPoolExecutor(5, 5, 5, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, service, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    //Only 3 servers return value.
    Assert.assertEquals(v.size(), 3);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    //No  response from 4th server
    Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
    Map<ServerInstance, Throwable> errorMap = fut.getError();
    Assert.assertEquals(errorMap.size(), 1, "One error");
    Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
    Thread.sleep(3000);
    pool.getStats().refresh();
    Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
}

Example 13 with ScheduledExecutorService

use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerHappy.

@Test
public void testMultipleServerHappy() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7071;
    int serverPort2 = 7072;
    int serverPort3 = 7073;
    int serverPort4 = 7074;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    Assert.assertEquals(v.size(), 4);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    b = v.get(serverInstance4);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_3_0");
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
}

Example 14 with ScheduledExecutorService

use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.

the class ThirdEyeAnomalyApplication method run.

@Override
public void run(final ThirdEyeAnomalyConfiguration config, final Environment environment) throws Exception {
    LOG.info("Starting ThirdeyeAnomalyApplication : Scheduler {} Worker {}", config.isScheduler(), config.isWorker());
    super.initDAOs();
    ThirdEyeCacheRegistry.initializeCaches(config);
    environment.lifecycle().manage(new Managed() {

        @Override
        public void start() throws Exception {
            if (config.isWorker()) {
                anomalyFunctionFactory = new AnomalyFunctionFactory(config.getFunctionConfigPath());
                alertFilterFactory = new AlertFilterFactory(config.getAlertFilterConfigPath());
                taskDriver = new TaskDriver(config, anomalyFunctionFactory, alertFilterFactory);
                taskDriver.start();
            }
            if (config.isScheduler()) {
                detectionJobScheduler = new DetectionJobScheduler();
                alertFilterFactory = new AlertFilterFactory(config.getAlertFilterConfigPath());
                alertFilterAutotuneFactory = new AlertFilterAutotuneFactory(config.getFilterAutotuneConfigPath());
                detectionJobScheduler.start();
                environment.jersey().register(new DetectionJobResource(detectionJobScheduler, alertFilterFactory, alertFilterAutotuneFactory));
                environment.jersey().register(new AnomalyFunctionResource(config.getFunctionConfigPath()));
            }
            if (config.isMonitor()) {
                monitorJobScheduler = new MonitorJobScheduler(config.getMonitorConfiguration());
                monitorJobScheduler.start();
            }
            if (config.isAlert()) {
                alertJobScheduler = new AlertJobScheduler();
                alertJobScheduler.start();
                // start alert scheduler v2
                alertJobSchedulerV2 = new AlertJobSchedulerV2();
                alertJobSchedulerV2.start();
                environment.jersey().register(new AlertJobResource(alertJobScheduler, emailConfigurationDAO));
            }
            if (config.isMerger()) {
                // anomalyFunctionFactory might have initiated if current machine is also a worker
                if (anomalyFunctionFactory == null) {
                    anomalyFunctionFactory = new AnomalyFunctionFactory(config.getFunctionConfigPath());
                }
                ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
                anomalyMergeExecutor = new AnomalyMergeExecutor(executorService, anomalyFunctionFactory);
                anomalyMergeExecutor.start();
            }
            if (config.isAutoload()) {
                autoLoadPinotMetricsService = new AutoLoadPinotMetricsService(config);
                autoLoadPinotMetricsService.start();
            }
            if (config.isDataCompleteness()) {
                dataCompletenessScheduler = new DataCompletenessScheduler();
                dataCompletenessScheduler.start();
            }
        }

        @Override
        public void stop() throws Exception {
            if (config.isWorker()) {
                taskDriver.stop();
            }
            if (config.isScheduler()) {
                detectionJobScheduler.shutdown();
            }
            if (config.isMonitor()) {
                monitorJobScheduler.stop();
            }
            if (config.isAlert()) {
                alertJobScheduler.shutdown();
                alertJobSchedulerV2.shutdown();
            }
            if (config.isMerger()) {
                anomalyMergeExecutor.stop();
            }
            if (config.isAutoload()) {
                autoLoadPinotMetricsService.shutdown();
            }
            if (config.isDataCompleteness()) {
                dataCompletenessScheduler.shutdown();
            }
        }
    });
}

Example 15 with ScheduledExecutorService

use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.

the class AnomalyReportDriver method main.

public static void main(String[] args) throws Exception {
    Runnable job = new Runnable() {

        @Override
        public void run() {
            try {
                System.out.println("Running report generation");
                runReportGenerator();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
    long delaySeconds = (getNextExecutionTime().getTime() - System.currentTimeMillis()) / 1000;
    // Repeat every 4 hours
    // scheduler.scheduleAtFixedRate(job, delaySeconds, TimeUnit.HOURS.toSeconds(4), TimeUnit.SECONDS);
    // Repeat every Day
    scheduler.scheduleAtFixedRate(job, delaySeconds, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS);
    System.out.println("Press q to quit");
    int ch = System.in.read();
    while (ch != 'q') {
        System.out.println("Press q to quit, next execution time : " + getNextExecutionTime());
        ch = System.in.read();
    }
    System.exit(-1);
}