use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.
the class ThirdEyeCacheRegistry method initCaches.
private static void initCaches(ThirdEyeConfiguration config) {
ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {
@Override
public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
LOGGER.info("Expired {}", notification.getKey().getPql());
}
};
// ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
// We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
// heap space.
long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
int resultSetCount = resultSetGroup.getResultSetCount();
int weight = 0;
for (int idx = 0; idx < resultSetCount; ++idx) {
com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
weight += (resultSet.getColumnCount() * resultSet.getRowCount());
}
return weight;
}).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
// CollectionMaxDataTime Cache
LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
// Query Cache
QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
cacheRegistry.registerQueryCache(queryCache);
// Dimension Filter cache
LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
// Dashboards cache
LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardsCache(dashboardsCache);
// Collections cache
CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
cacheRegistry.registerCollectionsCache(collectionsCache);
// DatasetConfig cache
LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
// MetricConfig cache
LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
cacheRegistry.registerMetricConfigCache(metricConfigCache);
// DashboardConfigs cache
LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}
use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.
the class AutoLoadPinotMetricsService method addNewDataset.
/**
* Adds a new dataset to the thirdeye database
* @param dataset
* @param schema
*/
private void addNewDataset(String dataset, Schema schema) throws Exception {
List<MetricFieldSpec> metricSpecs = schema.getMetricFieldSpecs();
// Create DatasetConfig
DatasetConfigDTO datasetConfigDTO = ConfigGenerator.generateDatasetConfig(dataset, schema);
LOG.info("Creating dataset for {}", dataset);
DAO_REGISTRY.getDatasetConfigDAO().save(datasetConfigDTO);
// Create MetricConfig
for (MetricFieldSpec metricFieldSpec : metricSpecs) {
MetricConfigDTO metricConfigDTO = ConfigGenerator.generateMetricConfig(metricFieldSpec, dataset);
LOG.info("Creating metric {} for {}", metricConfigDTO.getName(), dataset);
DAO_REGISTRY.getMetricConfigDAO().save(metricConfigDTO);
}
// Create Default DashboardConfig
List<Long> metricIds = ConfigGenerator.getMetricIdsFromMetricConfigs(DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset));
DashboardConfigDTO dashboardConfigDTO = ConfigGenerator.generateDefaultDashboardConfig(dataset, metricIds);
LOG.info("Creating default dashboard for dataset {}", dataset);
DAO_REGISTRY.getDashboardConfigDAO().save(dashboardConfigDTO);
}
use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.
the class AnomaliesResource method getAnomaliesByDashboardId.
/**
* Find anomalies by dashboard id
* @param startTime
* @param endTime
* @param dashboardId
* @param functionName
* @return
* @throws Exception
*/
@GET
@Path("search/dashboardId/{startTime}/{endTime}/{pageNumber}")
public AnomaliesWrapper getAnomaliesByDashboardId(@PathParam("startTime") Long startTime, @PathParam("endTime") Long endTime, @PathParam("pageNumber") int pageNumber, @QueryParam("dashboardId") String dashboardId, @QueryParam("functionName") String functionName) throws Exception {
DashboardConfigDTO dashboardConfig = dashboardConfigDAO.findById(Long.valueOf(dashboardId));
String metricIdsString = Joiner.on(COMMA_SEPARATOR).join(dashboardConfig.getMetricIds());
return getAnomaliesByMetricIds(startTime, endTime, pageNumber, metricIdsString, functionName);
}
use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.
the class Utils method getDashboards.
public static List<String> getDashboards(DashboardConfigManager dashboardConfigDAO, String collection) throws Exception {
List<DashboardConfigDTO> dashboardConfigs = dashboardConfigDAO.findActiveByDataset(collection);
List<String> dashboards = new ArrayList<>();
for (DashboardConfigDTO dashboardConfig : dashboardConfigs) {
dashboards.add(dashboardConfig.getName());
}
return dashboards;
}
use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.
the class EntityManagerResource method updateEntity.
@POST
public Response updateEntity(@QueryParam("entityType") String entityTypeStr, String jsonPayload) {
if (Strings.isNullOrEmpty(entityTypeStr)) {
throw new WebApplicationException("EntryType can not be null");
}
EntityType entityType = EntityType.valueOf(entityTypeStr);
try {
switch(entityType) {
case ANOMALY_FUNCTION:
AnomalyFunctionDTO anomalyFunctionDTO = OBJECT_MAPPER.readValue(jsonPayload, AnomalyFunctionDTO.class);
if (anomalyFunctionDTO.getId() == null) {
anomalyFunctionManager.save(anomalyFunctionDTO);
} else {
anomalyFunctionManager.update(anomalyFunctionDTO);
}
break;
case EMAIL_CONFIGURATION:
EmailConfigurationDTO emailConfigurationDTO = OBJECT_MAPPER.readValue(jsonPayload, EmailConfigurationDTO.class);
emailConfigurationManager.update(emailConfigurationDTO);
break;
case DASHBOARD_CONFIG:
DashboardConfigDTO dashboardConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, DashboardConfigDTO.class);
dashboardConfigManager.update(dashboardConfigDTO);
break;
case DATASET_CONFIG:
DatasetConfigDTO datasetConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, DatasetConfigDTO.class);
datasetConfigManager.update(datasetConfigDTO);
break;
case METRIC_CONFIG:
MetricConfigDTO metricConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, MetricConfigDTO.class);
metricConfigManager.update(metricConfigDTO);
break;
case OVERRIDE_CONFIG:
OverrideConfigDTO overrideConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, OverrideConfigDTO.class);
if (overrideConfigDTO.getId() == null) {
overrideConfigManager.save(overrideConfigDTO);
} else {
overrideConfigManager.update(overrideConfigDTO);
}
break;
case ALERT_CONFIG:
AlertConfigDTO alertConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, AlertConfigDTO.class);
if (alertConfigDTO.getId() == null) {
alertConfigManager.save(alertConfigDTO);
} else {
alertConfigManager.update(alertConfigDTO);
}
break;
}
} catch (IOException e) {
LOG.error("Error saving the entity with payload : " + jsonPayload, e);
throw new WebApplicationException(e);
}
return Response.ok().build();
}
Aggregations