use of com.linkedin.thirdeye.client.cache.ResultSetGroupCacheLoader in project pinot by linkedin.
the class ThirdEyeCacheRegistry method initCaches.
private static void initCaches(ThirdEyeConfiguration config) {
ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {
@Override
public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
LOGGER.info("Expired {}", notification.getKey().getPql());
}
};
// ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
// We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
// heap space.
long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
int resultSetCount = resultSetGroup.getResultSetCount();
int weight = 0;
for (int idx = 0; idx < resultSetCount; ++idx) {
com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
weight += (resultSet.getColumnCount() * resultSet.getRowCount());
}
return weight;
}).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
// CollectionMaxDataTime Cache
LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
// Query Cache
QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
cacheRegistry.registerQueryCache(queryCache);
// Dimension Filter cache
LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
// Dashboards cache
LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardsCache(dashboardsCache);
// Collections cache
CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
cacheRegistry.registerCollectionsCache(collectionsCache);
// DatasetConfig cache
LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
// MetricConfig cache
LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
cacheRegistry.registerMetricConfigCache(metricConfigCache);
// DashboardConfigs cache
LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}
Aggregations