Search in sources :

Example 1 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class ThirdEyeCacheRegistry method initCaches.

private static void initCaches(ThirdEyeConfiguration config) {
    ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
    RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {

        @Override
        public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
            LOGGER.info("Expired {}", notification.getKey().getPql());
        }
    };
    // ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
    // We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
    // heap space.
    long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
    LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
        int resultSetCount = resultSetGroup.getResultSetCount();
        int weight = 0;
        for (int idx = 0; idx < resultSetCount; ++idx) {
            com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
            weight += (resultSet.getColumnCount() * resultSet.getRowCount());
        }
        return weight;
    }).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
    cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
    LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
    // CollectionMaxDataTime Cache
    LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
    cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
    // Query Cache
    QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
    cacheRegistry.registerQueryCache(queryCache);
    // Dimension Filter cache
    LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
    cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
    // Dashboards cache
    LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
    cacheRegistry.registerDashboardsCache(dashboardsCache);
    // Collections cache
    CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
    cacheRegistry.registerCollectionsCache(collectionsCache);
    // DatasetConfig cache
    LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
    cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
    // MetricConfig cache
    LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
    cacheRegistry.registerMetricConfigCache(metricConfigCache);
    // DashboardConfigs cache
    LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
    cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}
Also used : ResultSetGroup(com.linkedin.pinot.client.ResultSetGroup) MetricConfigDTO(com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO) LoadingCache(com.google.common.cache.LoadingCache) DimensionFiltersCacheLoader(com.linkedin.thirdeye.client.cache.DimensionFiltersCacheLoader) PinotQuery(com.linkedin.thirdeye.client.pinot.PinotQuery) DashboardConfigManager(com.linkedin.thirdeye.datalayer.bao.DashboardConfigManager) LoggerFactory(org.slf4j.LoggerFactory) ThirdEyeConfiguration(com.linkedin.thirdeye.common.ThirdEyeConfiguration) DatasetConfigManager(com.linkedin.thirdeye.datalayer.bao.DatasetConfigManager) MetricDataset(com.linkedin.thirdeye.client.cache.MetricDataset) MetricConfigManager(com.linkedin.thirdeye.datalayer.bao.MetricConfigManager) PinotThirdEyeClient(com.linkedin.thirdeye.client.pinot.PinotThirdEyeClient) QueryCache(com.linkedin.thirdeye.client.cache.QueryCache) CollectionMaxDataTimeCacheLoader(com.linkedin.thirdeye.client.cache.CollectionMaxDataTimeCacheLoader) MetricConfigCacheLoader(com.linkedin.thirdeye.client.cache.MetricConfigCacheLoader) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) CacheResource(com.linkedin.thirdeye.dashboard.resources.CacheResource) DashboardConfigCacheLoader(com.linkedin.thirdeye.client.cache.DashboardConfigCacheLoader) RemovalNotification(com.google.common.cache.RemovalNotification) CollectionsCache(com.linkedin.thirdeye.client.cache.CollectionsCache) Logger(org.slf4j.Logger) PinotThirdEyeClientConfig(com.linkedin.thirdeye.client.pinot.PinotThirdEyeClientConfig) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) DatasetConfigCacheLoader(com.linkedin.thirdeye.client.cache.DatasetConfigCacheLoader) List(java.util.List) DashboardsCacheLoader(com.linkedin.thirdeye.client.cache.DashboardsCacheLoader) ResultSetGroupCacheLoader(com.linkedin.thirdeye.client.cache.ResultSetGroupCacheLoader) DashboardConfigDTO(com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO) RemovalListener(com.google.common.cache.RemovalListener) CacheBuilder(com.google.common.cache.CacheBuilder) DatasetConfigDTO(com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO) ResultSetGroupCacheLoader(com.linkedin.thirdeye.client.cache.ResultSetGroupCacheLoader) QueryCache(com.linkedin.thirdeye.client.cache.QueryCache) DatasetConfigDTO(com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO) DatasetConfigCacheLoader(com.linkedin.thirdeye.client.cache.DatasetConfigCacheLoader) DimensionFiltersCacheLoader(com.linkedin.thirdeye.client.cache.DimensionFiltersCacheLoader) List(java.util.List) MetricConfigDTO(com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO) DashboardConfigCacheLoader(com.linkedin.thirdeye.client.cache.DashboardConfigCacheLoader) RemovalListener(com.google.common.cache.RemovalListener) ResultSetGroup(com.linkedin.pinot.client.ResultSetGroup) DashboardsCacheLoader(com.linkedin.thirdeye.client.cache.DashboardsCacheLoader) MetricDataset(com.linkedin.thirdeye.client.cache.MetricDataset) CollectionMaxDataTimeCacheLoader(com.linkedin.thirdeye.client.cache.CollectionMaxDataTimeCacheLoader) PinotQuery(com.linkedin.thirdeye.client.pinot.PinotQuery) MetricConfigCacheLoader(com.linkedin.thirdeye.client.cache.MetricConfigCacheLoader) RemovalNotification(com.google.common.cache.RemovalNotification) CollectionsCache(com.linkedin.thirdeye.client.cache.CollectionsCache)

Example 2 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class AutoLoadPinotMetricsService method addNewDataset.

/**
   * Adds a new dataset to the thirdeye database
   * @param dataset
   * @param schema
   */
private void addNewDataset(String dataset, Schema schema) throws Exception {
    List<MetricFieldSpec> metricSpecs = schema.getMetricFieldSpecs();
    // Create DatasetConfig
    DatasetConfigDTO datasetConfigDTO = ConfigGenerator.generateDatasetConfig(dataset, schema);
    LOG.info("Creating dataset for {}", dataset);
    DAO_REGISTRY.getDatasetConfigDAO().save(datasetConfigDTO);
    // Create MetricConfig
    for (MetricFieldSpec metricFieldSpec : metricSpecs) {
        MetricConfigDTO metricConfigDTO = ConfigGenerator.generateMetricConfig(metricFieldSpec, dataset);
        LOG.info("Creating metric {} for {}", metricConfigDTO.getName(), dataset);
        DAO_REGISTRY.getMetricConfigDAO().save(metricConfigDTO);
    }
    // Create Default DashboardConfig
    List<Long> metricIds = ConfigGenerator.getMetricIdsFromMetricConfigs(DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset));
    DashboardConfigDTO dashboardConfigDTO = ConfigGenerator.generateDefaultDashboardConfig(dataset, metricIds);
    LOG.info("Creating default dashboard for dataset {}", dataset);
    DAO_REGISTRY.getDashboardConfigDAO().save(dashboardConfigDTO);
}
Also used : DatasetConfigDTO(com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO) MetricConfigDTO(com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO) MetricFieldSpec(com.linkedin.pinot.common.data.MetricFieldSpec) DashboardConfigDTO(com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO)

Example 3 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class AnomaliesResource method getAnomaliesByDashboardId.

/**
   * Find anomalies by dashboard id
   * @param startTime
   * @param endTime
   * @param dashboardId
   * @param functionName
   * @return
   * @throws Exception
   */
@GET
@Path("search/dashboardId/{startTime}/{endTime}/{pageNumber}")
public AnomaliesWrapper getAnomaliesByDashboardId(@PathParam("startTime") Long startTime, @PathParam("endTime") Long endTime, @PathParam("pageNumber") int pageNumber, @QueryParam("dashboardId") String dashboardId, @QueryParam("functionName") String functionName) throws Exception {
    DashboardConfigDTO dashboardConfig = dashboardConfigDAO.findById(Long.valueOf(dashboardId));
    String metricIdsString = Joiner.on(COMMA_SEPARATOR).join(dashboardConfig.getMetricIds());
    return getAnomaliesByMetricIds(startTime, endTime, pageNumber, metricIdsString, functionName);
}
Also used : DashboardConfigDTO(com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 4 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class Utils method getDashboards.

public static List<String> getDashboards(DashboardConfigManager dashboardConfigDAO, String collection) throws Exception {
    List<DashboardConfigDTO> dashboardConfigs = dashboardConfigDAO.findActiveByDataset(collection);
    List<String> dashboards = new ArrayList<>();
    for (DashboardConfigDTO dashboardConfig : dashboardConfigs) {
        dashboards.add(dashboardConfig.getName());
    }
    return dashboards;
}
Also used : ArrayList(java.util.ArrayList) DashboardConfigDTO(com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO)

Example 5 with DashboardConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO in project pinot by linkedin.

the class EntityManagerResource method updateEntity.

@POST
public Response updateEntity(@QueryParam("entityType") String entityTypeStr, String jsonPayload) {
    if (Strings.isNullOrEmpty(entityTypeStr)) {
        throw new WebApplicationException("EntryType can not be null");
    }
    EntityType entityType = EntityType.valueOf(entityTypeStr);
    try {
        switch(entityType) {
            case ANOMALY_FUNCTION:
                AnomalyFunctionDTO anomalyFunctionDTO = OBJECT_MAPPER.readValue(jsonPayload, AnomalyFunctionDTO.class);
                if (anomalyFunctionDTO.getId() == null) {
                    anomalyFunctionManager.save(anomalyFunctionDTO);
                } else {
                    anomalyFunctionManager.update(anomalyFunctionDTO);
                }
                break;
            case EMAIL_CONFIGURATION:
                EmailConfigurationDTO emailConfigurationDTO = OBJECT_MAPPER.readValue(jsonPayload, EmailConfigurationDTO.class);
                emailConfigurationManager.update(emailConfigurationDTO);
                break;
            case DASHBOARD_CONFIG:
                DashboardConfigDTO dashboardConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, DashboardConfigDTO.class);
                dashboardConfigManager.update(dashboardConfigDTO);
                break;
            case DATASET_CONFIG:
                DatasetConfigDTO datasetConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, DatasetConfigDTO.class);
                datasetConfigManager.update(datasetConfigDTO);
                break;
            case METRIC_CONFIG:
                MetricConfigDTO metricConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, MetricConfigDTO.class);
                metricConfigManager.update(metricConfigDTO);
                break;
            case OVERRIDE_CONFIG:
                OverrideConfigDTO overrideConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, OverrideConfigDTO.class);
                if (overrideConfigDTO.getId() == null) {
                    overrideConfigManager.save(overrideConfigDTO);
                } else {
                    overrideConfigManager.update(overrideConfigDTO);
                }
                break;
            case ALERT_CONFIG:
                AlertConfigDTO alertConfigDTO = OBJECT_MAPPER.readValue(jsonPayload, AlertConfigDTO.class);
                if (alertConfigDTO.getId() == null) {
                    alertConfigManager.save(alertConfigDTO);
                } else {
                    alertConfigManager.update(alertConfigDTO);
                }
                break;
        }
    } catch (IOException e) {
        LOG.error("Error saving the entity with payload : " + jsonPayload, e);
        throw new WebApplicationException(e);
    }
    return Response.ok().build();
}
Also used : DatasetConfigDTO(com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO) MetricConfigDTO(com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO) OverrideConfigDTO(com.linkedin.thirdeye.datalayer.dto.OverrideConfigDTO) WebApplicationException(javax.ws.rs.WebApplicationException) AnomalyFunctionDTO(com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO) EmailConfigurationDTO(com.linkedin.thirdeye.datalayer.dto.EmailConfigurationDTO) AlertConfigDTO(com.linkedin.thirdeye.datalayer.dto.AlertConfigDTO) IOException(java.io.IOException) DashboardConfigDTO(com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO) POST(javax.ws.rs.POST)

Aggregations

DashboardConfigDTO (com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO)18 ArrayList (java.util.ArrayList)9 MetricConfigDTO (com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO)8 MetricFieldSpec (com.linkedin.pinot.common.data.MetricFieldSpec)4 DatasetConfigDTO (com.linkedin.thirdeye.datalayer.dto.DatasetConfigDTO)4 DashboardConfigBean (com.linkedin.thirdeye.datalayer.pojo.DashboardConfigBean)4 GET (javax.ws.rs.GET)4 Path (javax.ws.rs.Path)4 Predicate (com.linkedin.thirdeye.datalayer.util.Predicate)3 IOException (java.io.IOException)2 Test (org.testng.annotations.Test)2 CacheBuilder (com.google.common.cache.CacheBuilder)1 LoadingCache (com.google.common.cache.LoadingCache)1 RemovalListener (com.google.common.cache.RemovalListener)1 RemovalNotification (com.google.common.cache.RemovalNotification)1 ResultSetGroup (com.linkedin.pinot.client.ResultSetGroup)1 DimensionFieldSpec (com.linkedin.pinot.common.data.DimensionFieldSpec)1 FieldSpec (com.linkedin.pinot.common.data.FieldSpec)1 TimeGranularitySpec (com.linkedin.pinot.common.data.TimeGranularitySpec)1 MetricExpression (com.linkedin.thirdeye.client.MetricExpression)1