use of com.linkedin.thirdeye.common.ThirdEyeConfiguration in project pinot by linkedin.
the class ThirdEyeCacheRegistry method initCaches.
private static void initCaches(ThirdEyeConfiguration config) {
ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {
@Override
public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
LOGGER.info("Expired {}", notification.getKey().getPql());
}
};
// ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
// We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
// heap space.
long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
int resultSetCount = resultSetGroup.getResultSetCount();
int weight = 0;
for (int idx = 0; idx < resultSetCount; ++idx) {
com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
weight += (resultSet.getColumnCount() * resultSet.getRowCount());
}
return weight;
}).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
// CollectionMaxDataTime Cache
LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
// Query Cache
QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
cacheRegistry.registerQueryCache(queryCache);
// Dimension Filter cache
LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
// Dashboards cache
LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardsCache(dashboardsCache);
// Collections cache
CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
cacheRegistry.registerCollectionsCache(collectionsCache);
// DatasetConfig cache
LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
// MetricConfig cache
LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
cacheRegistry.registerMetricConfigCache(metricConfigCache);
// DashboardConfigs cache
LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}
use of com.linkedin.thirdeye.common.ThirdEyeConfiguration in project pinot by linkedin.
the class PinotThirdEyeSummaryClient method main.
public static void main(String[] args) throws Exception {
String oFileName = "Cube.json";
// An interesting data set that difficult to tell because too many dark reds and blues (Granularity: DAYS)
// String collection = "thirdeyeKbmi";
// String metricName = "pageViews";
// DateTime baselineStart = new DateTime(1467788400000L);
// DateTime baselineEnd = new DateTime(1469862000000L);
// DateTime currentStart = new DateTime(1468393200000L);
// DateTime currentEnd = new DateTime(1470466800000L);
// An interesting data set that difficult to tell because most cells are light red or blue (Granularity: HOURS)
// String collection = "thirdeyeKbmi";
// String metricName = "mobilePageViews";
// DateTime baselineStart = new DateTime(1469628000000L);
// DateTime baselineEnd = new DateTime(1469714400000L);
// DateTime currentStart = new DateTime(1470232800000L);
// DateTime currentEnd = new DateTime(1470319200000L);
// A migration of Asia connections from other data center to lsg data center (Granularity: DAYS)
// Most contributors: India and China
// String collection = "thirdeyeAbook";
// String metricName = "totalFlows";
// DateTime baselineStart = new DateTime(2016, 7, 11, 00, 00);
// DateTime baselineEnd = new DateTime(2016, 7, 12, 00, 00);
// DateTime currentStart = new DateTime(2016, 7, 18, 00, 00);
// DateTime currentEnd = new DateTime(2016, 7, 19, 00, 00);
// National Holidays in India and several countries in Europe and Latin America. (Granularity: DAYS)
String collection = "thirdeyeKbmi";
String metricName = "desktopPageViews";
DateTime baselineStart = new DateTime(1470589200000L);
DateTime baselineEnd = new DateTime(1470675600000L);
DateTime currentStart = new DateTime(1471194000000L);
DateTime currentEnd = new DateTime(1471280400000L);
// String collection = "ptrans_additive";
// String metricName = "txProcessTime/__COUNT";
// DateTime baselineStart = new DateTime(1470938400000L);
// DateTime baselineEnd = new DateTime(1471024800000L);
// DateTime currentStart = new DateTime(1471543200000L);
// DateTime currentEnd = new DateTime(1471629600000L);
// Create ThirdEye client
List<String> argList = new ArrayList<String>(Arrays.asList(args));
if (argList.size() == 1) {
argList.add(0, "server");
}
int lastIndex = argList.size() - 1;
String thirdEyeConfigDir = argList.get(lastIndex);
String persistenceConfig = thirdEyeConfigDir + "/persistence.yml";
DaoProviderUtil.init(new File(persistenceConfig));
ThirdEyeConfiguration thirdEyeConfig = new ThirdEyeDashboardConfiguration();
thirdEyeConfig.setWhitelistCollections(collection);
thirdEyeConfig.setRootDir(thirdEyeConfigDir);
ThirdEyeCacheRegistry.initializeCaches(thirdEyeConfig);
ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metricName, MetricAggFunction.SUM, collection);
System.out.println(metricExpressions);
OLAPDataBaseClient pinotClient = new PinotThirdEyeSummaryClient(CACHE_REGISTRY_INSTANCE.getQueryCache());
pinotClient.setCollection(collection);
pinotClient.setMetricExpression(metricExpressions.get(0));
pinotClient.setCurrentStartInclusive(currentStart);
pinotClient.setCurrentEndExclusive(currentEnd);
pinotClient.setBaselineStartInclusive(baselineStart);
pinotClient.setBaselineEndExclusive(baselineEnd);
List<List<String>> hierarchies = new ArrayList<>();
hierarchies.add(Lists.newArrayList("continent", "countryCode"));
hierarchies.add(Lists.newArrayList("browser_name", "browser_version"));
Dimensions dimensions;
try {
dimensions = new Dimensions(Utils.getSchemaDimensionNames(collection));
} catch (Exception e1) {
System.out.println("Failed to get dimensions names of the collection: " + collection);
String[] dimensionNames = { "browserName", "continent", "countryCode", "deviceName", "environment", "locale", "osName", "pageKey", "service", "sourceApp" };
System.out.println("Default dimension names are used:" + dimensionNames);
dimensions = new Dimensions(Lists.newArrayList(dimensionNames));
}
int maxDimensionSize = 3;
// Build the cube for computing the summary
Cube initCube = new Cube();
initCube.buildWithAutoDimensionOrder(pinotClient, dimensions, maxDimensionSize, hierarchies);
// initCube.buildWithManualDimensionOrder(pinotClient, dimensions);
int answerSize = 10;
boolean oneSideErrors = true;
Summary summary = new Summary(initCube);
System.out.println(summary.computeSummary(answerSize, oneSideErrors, maxDimensionSize));
try {
initCube.toJson(oFileName);
Cube cube = Cube.fromJson(oFileName);
System.out.println("Restored Cube:");
System.out.println(cube);
summary = new Summary(cube);
System.out.println(summary.computeSummary(answerSize, oneSideErrors, maxDimensionSize));
} catch (IOException e) {
System.err.println("WARN: Unable to save the cube to the file: " + oFileName);
e.printStackTrace();
}
// closing
System.exit(0);
}
Aggregations