use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class ScatterGatherTest method testMultipleServerHappy.
@Test
public void testMultipleServerHappy() throws Exception {
MetricsRegistry registry = new MetricsRegistry();
// Server start
int serverPort1 = 7071;
int serverPort2 = 7072;
int serverPort3 = 7073;
int serverPort4 = 7074;
NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1), null);
Thread t1 = new Thread(server1);
Thread t2 = new Thread(server2);
Thread t3 = new Thread(server3);
Thread t4 = new Thread(server4);
t1.start();
t2.start();
t3.start();
t4.start();
//Client setup
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
rm.setPool(pool);
SegmentIdSet pg1 = new SegmentIdSet();
pg1.addSegment(new SegmentId("0"));
SegmentIdSet pg2 = new SegmentIdSet();
pg2.addSegment(new SegmentId("1"));
SegmentIdSet pg3 = new SegmentIdSet();
pg3.addSegment(new SegmentId("2"));
SegmentIdSet pg4 = new SegmentIdSet();
pg4.addSegment(new SegmentId("3"));
ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
pgMap.put(serverInstance1, pg1);
pgMap.put(serverInstance2, pg2);
pgMap.put(serverInstance3, pg3);
pgMap.put(serverInstance4, pg4);
String request1 = "request_0";
String request2 = "request_1";
String request3 = "request_2";
String request4 = "request_3";
Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
pgMapStr.put(pg1, request1);
pgMapStr.put(pg2, request2);
pgMapStr.put(pg3, request3);
pgMapStr.put(pg4, request4);
ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
Map<ServerInstance, ByteBuf> v = fut.get();
Assert.assertEquals(v.size(), 4);
ByteBuf b = v.get(serverInstance1);
byte[] b2 = new byte[b.readableBytes()];
b.readBytes(b2);
String response = new String(b2);
Assert.assertEquals(response, "response_0_0");
b = v.get(serverInstance2);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_1_0");
b = v.get(serverInstance3);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_2_0");
b = v.get(serverInstance4);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_3_0");
server1.shutdownGracefully();
server2.shutdownGracefully();
server3.shutdownGracefully();
server4.shutdownGracefully();
pool.shutdown();
service.shutdown();
eventLoopGroup.shutdownGracefully();
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class ThirdEyeAnomalyApplication method run.
@Override
public void run(final ThirdEyeAnomalyConfiguration config, final Environment environment) throws Exception {
LOG.info("Starting ThirdeyeAnomalyApplication : Scheduler {} Worker {}", config.isScheduler(), config.isWorker());
super.initDAOs();
ThirdEyeCacheRegistry.initializeCaches(config);
environment.lifecycle().manage(new Managed() {
@Override
public void start() throws Exception {
if (config.isWorker()) {
anomalyFunctionFactory = new AnomalyFunctionFactory(config.getFunctionConfigPath());
alertFilterFactory = new AlertFilterFactory(config.getAlertFilterConfigPath());
taskDriver = new TaskDriver(config, anomalyFunctionFactory, alertFilterFactory);
taskDriver.start();
}
if (config.isScheduler()) {
detectionJobScheduler = new DetectionJobScheduler();
alertFilterFactory = new AlertFilterFactory(config.getAlertFilterConfigPath());
alertFilterAutotuneFactory = new AlertFilterAutotuneFactory(config.getFilterAutotuneConfigPath());
detectionJobScheduler.start();
environment.jersey().register(new DetectionJobResource(detectionJobScheduler, alertFilterFactory, alertFilterAutotuneFactory));
environment.jersey().register(new AnomalyFunctionResource(config.getFunctionConfigPath()));
}
if (config.isMonitor()) {
monitorJobScheduler = new MonitorJobScheduler(config.getMonitorConfiguration());
monitorJobScheduler.start();
}
if (config.isAlert()) {
alertJobScheduler = new AlertJobScheduler();
alertJobScheduler.start();
// start alert scheduler v2
alertJobSchedulerV2 = new AlertJobSchedulerV2();
alertJobSchedulerV2.start();
environment.jersey().register(new AlertJobResource(alertJobScheduler, emailConfigurationDAO));
}
if (config.isMerger()) {
// anomalyFunctionFactory might have initiated if current machine is also a worker
if (anomalyFunctionFactory == null) {
anomalyFunctionFactory = new AnomalyFunctionFactory(config.getFunctionConfigPath());
}
ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
anomalyMergeExecutor = new AnomalyMergeExecutor(executorService, anomalyFunctionFactory);
anomalyMergeExecutor.start();
}
if (config.isAutoload()) {
autoLoadPinotMetricsService = new AutoLoadPinotMetricsService(config);
autoLoadPinotMetricsService.start();
}
if (config.isDataCompleteness()) {
dataCompletenessScheduler = new DataCompletenessScheduler();
dataCompletenessScheduler.start();
}
}
@Override
public void stop() throws Exception {
if (config.isWorker()) {
taskDriver.stop();
}
if (config.isScheduler()) {
detectionJobScheduler.shutdown();
}
if (config.isMonitor()) {
monitorJobScheduler.stop();
}
if (config.isAlert()) {
alertJobScheduler.shutdown();
alertJobSchedulerV2.shutdown();
}
if (config.isMerger()) {
anomalyMergeExecutor.stop();
}
if (config.isAutoload()) {
autoLoadPinotMetricsService.shutdown();
}
if (config.isDataCompleteness()) {
dataCompletenessScheduler.shutdown();
}
}
});
}
use of java.util.concurrent.ScheduledExecutorService in project pinot by linkedin.
the class AnomalyReportDriver method main.
public static void main(String[] args) throws Exception {
Runnable job = new Runnable() {
@Override
public void run() {
try {
System.out.println("Running report generation");
runReportGenerator();
} catch (Exception e) {
e.printStackTrace();
}
}
};
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
long delaySeconds = (getNextExecutionTime().getTime() - System.currentTimeMillis()) / 1000;
// Repeat every 4 hours
// scheduler.scheduleAtFixedRate(job, delaySeconds, TimeUnit.HOURS.toSeconds(4), TimeUnit.SECONDS);
// Repeat every Day
scheduler.scheduleAtFixedRate(job, delaySeconds, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS);
System.out.println("Press q to quit");
int ch = System.in.read();
while (ch != 'q') {
System.out.println("Press q to quit, next execution time : " + getNextExecutionTime());
ch = System.in.read();
}
System.exit(-1);
}
use of java.util.concurrent.ScheduledExecutorService in project rest.li by linkedin.
the class SimpleLoadBalancerTest method testLoadBalancerSmoke.
@Test(groups = { "small", "back-end" })
public void testLoadBalancerSmoke() throws URISyntaxException, ServiceUnavailableException, InterruptedException, ExecutionException {
for (int tryAgain = 0; tryAgain < 1000; ++tryAgain) {
Map<String, LoadBalancerStrategyFactory<? extends LoadBalancerStrategy>> loadBalancerStrategyFactories = new HashMap<String, LoadBalancerStrategyFactory<? extends LoadBalancerStrategy>>();
Map<String, TransportClientFactory> clientFactories = new HashMap<String, TransportClientFactory>();
List<String> prioritizedSchemes = new ArrayList<String>();
MockStore<ServiceProperties> serviceRegistry = new MockStore<ServiceProperties>();
MockStore<ClusterProperties> clusterRegistry = new MockStore<ClusterProperties>();
MockStore<UriProperties> uriRegistry = new MockStore<UriProperties>();
ScheduledExecutorService executorService = new SynchronousExecutorService();
//loadBalancerStrategyFactories.put("rr", new RandomLoadBalancerStrategyFactory());
loadBalancerStrategyFactories.put("degrader", new DegraderLoadBalancerStrategyFactoryV3());
// PrpcClientFactory();
// new
clientFactories.put("http", new DoNothingClientFactory());
// HttpClientFactory();
SimpleLoadBalancerState state = new SimpleLoadBalancerState(executorService, uriRegistry, clusterRegistry, serviceRegistry, clientFactories, loadBalancerStrategyFactories);
SimpleLoadBalancer loadBalancer = new SimpleLoadBalancer(state, 5, TimeUnit.SECONDS);
FutureCallback<None> balancerCallback = new FutureCallback<None>();
loadBalancer.start(balancerCallback);
balancerCallback.get();
URI uri1 = URI.create("http://test.qa1.com:1234");
URI uri2 = URI.create("http://test.qa2.com:2345");
URI uri3 = URI.create("http://test.qa3.com:6789");
Map<Integer, PartitionData> partitionData = new HashMap<Integer, PartitionData>(1);
partitionData.put(DefaultPartitionAccessor.DEFAULT_PARTITION_ID, new PartitionData(1d));
Map<URI, Map<Integer, PartitionData>> uriData = new HashMap<URI, Map<Integer, PartitionData>>(3);
uriData.put(uri1, partitionData);
uriData.put(uri2, partitionData);
uriData.put(uri3, partitionData);
prioritizedSchemes.add("http");
clusterRegistry.put("cluster-1", new ClusterProperties("cluster-1"));
serviceRegistry.put("foo", new ServiceProperties("foo", "cluster-1", "/foo", Arrays.asList("degrader"), Collections.<String, Object>emptyMap(), null, null, prioritizedSchemes, null));
uriRegistry.put("cluster-1", new UriProperties("cluster-1", uriData));
URI expectedUri1 = URI.create("http://test.qa1.com:1234/foo");
URI expectedUri2 = URI.create("http://test.qa2.com:2345/foo");
URI expectedUri3 = URI.create("http://test.qa3.com:6789/foo");
Set<URI> expectedUris = new HashSet<URI>();
expectedUris.add(expectedUri1);
expectedUris.add(expectedUri2);
expectedUris.add(expectedUri3);
for (int i = 0; i < 100; ++i) {
RewriteClient client = (RewriteClient) loadBalancer.getClient(new URIRequest("d2://foo/52"), new RequestContext());
assertTrue(expectedUris.contains(client.getUri()));
assertEquals(client.getUri().getScheme(), "http");
}
final CountDownLatch latch = new CountDownLatch(1);
PropertyEventShutdownCallback callback = new PropertyEventShutdownCallback() {
@Override
public void done() {
latch.countDown();
}
};
state.shutdown(callback);
if (!latch.await(60, TimeUnit.SECONDS)) {
fail("unable to shutdown state");
}
executorService.shutdownNow();
assertTrue(executorService.isShutdown(), "ExecutorService should have shut down!");
}
}
use of java.util.concurrent.ScheduledExecutorService in project rest.li by linkedin.
the class SimpleLoadBalancerTest method testLoadBalancerDropRate.
/**
* This test simulates dropping requests by playing with OverrideDropRate in config
*
*/
@Test(groups = { "small", "back-end" })
public void testLoadBalancerDropRate() throws ServiceUnavailableException, ExecutionException, InterruptedException {
final int RETRY = 10;
for (int tryAgain = 0; tryAgain < RETRY; ++tryAgain) {
Map<String, LoadBalancerStrategyFactory<? extends LoadBalancerStrategy>> loadBalancerStrategyFactories = new HashMap<String, LoadBalancerStrategyFactory<? extends LoadBalancerStrategy>>();
Map<String, TransportClientFactory> clientFactories = new HashMap<String, TransportClientFactory>();
List<String> prioritizedSchemes = new ArrayList<String>();
MockStore<ServiceProperties> serviceRegistry = new MockStore<ServiceProperties>();
MockStore<ClusterProperties> clusterRegistry = new MockStore<ClusterProperties>();
MockStore<UriProperties> uriRegistry = new MockStore<UriProperties>();
ScheduledExecutorService executorService = new SynchronousExecutorService();
//loadBalancerStrategyFactories.put("rr", new RandomLoadBalancerStrategyFactory());
loadBalancerStrategyFactories.put("degrader", new DegraderLoadBalancerStrategyFactoryV3());
// PrpcClientFactory();
// new
clientFactories.put("http", new DoNothingClientFactory());
// HttpClientFactory();
SimpleLoadBalancerState state = new SimpleLoadBalancerState(executorService, uriRegistry, clusterRegistry, serviceRegistry, clientFactories, loadBalancerStrategyFactories);
SimpleLoadBalancer loadBalancer = new SimpleLoadBalancer(state, 5, TimeUnit.SECONDS);
FutureCallback<None> balancerCallback = new FutureCallback<None>();
loadBalancer.start(balancerCallback);
balancerCallback.get();
URI uri1 = URI.create("http://test.qa1.com:1234");
URI uri2 = URI.create("http://test.qa2.com:2345");
URI uri3 = URI.create("http://test.qa3.com:6789");
Map<Integer, PartitionData> partitionData = new HashMap<Integer, PartitionData>(1);
partitionData.put(DefaultPartitionAccessor.DEFAULT_PARTITION_ID, new PartitionData(1d));
Map<URI, Map<Integer, PartitionData>> uriData = new HashMap<URI, Map<Integer, PartitionData>>(3);
uriData.put(uri1, partitionData);
uriData.put(uri2, partitionData);
uriData.put(uri3, partitionData);
prioritizedSchemes.add("http");
clusterRegistry.put("cluster-1", new ClusterProperties("cluster-1"));
serviceRegistry.put("foo", new ServiceProperties("foo", "cluster-1", "/foo", Arrays.asList("degrader"), Collections.<String, Object>emptyMap(), null, null, prioritizedSchemes, null));
uriRegistry.put("cluster-1", new UriProperties("cluster-1", uriData));
URI expectedUri1 = URI.create("http://test.qa1.com:1234/foo");
URI expectedUri2 = URI.create("http://test.qa2.com:2345/foo");
URI expectedUri3 = URI.create("http://test.qa3.com:6789/foo");
Set<URI> expectedUris = new HashSet<URI>();
expectedUris.add(expectedUri1);
expectedUris.add(expectedUri2);
expectedUris.add(expectedUri3);
Random random = new Random();
for (int i = 0; i < 100; ++i) {
try {
RewriteClient client = (RewriteClient) loadBalancer.getClient(new URIRequest("d2://foo/52"), new RequestContext());
TrackerClient tClient = (TrackerClient) client.getWrappedClient();
DegraderImpl degrader = (DegraderImpl) tClient.getDegrader(DefaultPartitionAccessor.DEFAULT_PARTITION_ID);
DegraderImpl.Config cfg = new DegraderImpl.Config(degrader.getConfig());
// Change DropRate to 0.0 at the rate of 1/3
cfg.setOverrideDropRate((random.nextInt(2) == 0) ? 1.0 : 0.0);
degrader.setConfig(cfg);
assertTrue(expectedUris.contains(client.getUri()));
assertEquals(client.getUri().getScheme(), "http");
} catch (ServiceUnavailableException e) {
assertTrue(e.toString().contains("in a bad state (high latency/high error)"));
}
}
final CountDownLatch latch = new CountDownLatch(1);
PropertyEventShutdownCallback callback = new PropertyEventShutdownCallback() {
@Override
public void done() {
latch.countDown();
}
};
state.shutdown(callback);
if (!latch.await(60, TimeUnit.SECONDS)) {
fail("unable to shutdown state");
}
executorService.shutdownNow();
assertTrue(executorService.isShutdown(), "ExecutorService should have shut down!");
}
}
Aggregations