Examples with MetricConfigDTO com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO used on opensource projects

Example 1 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class AnomalyReportGenerator method getAnomaliesForMetrics.

public List<MergedAnomalyResultDTO> getAnomaliesForMetrics(List<String> metrics, long startTime, long endTime) {
    List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
    LOG.info("fetching anomalies for metrics : " + metrics);
    for (String metric : metrics) {
        List<MetricConfigDTO> metricConfigDTOList = metricConfigManager.findByMetricName(metric);
        for (MetricConfigDTO metricConfigDTO : metricConfigDTOList) {
            List<MergedAnomalyResultDTO> results = anomalyResultManager.findByCollectionMetricTime(metricConfigDTO.getDataset(), metric, startTime, endTime, false);
            LOG.info("Found {} result for metric {}", results.size(), metric);
            anomalies.addAll(results);
        }
    }
    return anomalies;
}

Example 2 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class MetricConfigCacheLoader method load.

@Override
public MetricConfigDTO load(MetricDataset metricDataset) throws Exception {
    LOGGER.info("Loading MetricConfigCache for metric {} of {}", metricDataset.getMetricName(), metricDataset.getDataset());
    MetricConfigDTO metricConfig = metricConfigDAO.findByMetricAndDataset(metricDataset.getMetricName(), metricDataset.getDataset());
    return metricConfig;
}

Example 3 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class ThirdEyeCacheRegistry method initCaches.

private static void initCaches(ThirdEyeConfiguration config) {
    ThirdEyeCacheRegistry cacheRegistry = ThirdEyeCacheRegistry.getInstance();
    RemovalListener<PinotQuery, ResultSetGroup> listener = new RemovalListener<PinotQuery, ResultSetGroup>() {

        @Override
        public void onRemoval(RemovalNotification<PinotQuery, ResultSetGroup> notification) {
            LOGGER.info("Expired {}", notification.getKey().getPql());
        }
    };
    // ResultSetGroup Cache. The size of this cache is limited by the total number of buckets in all ResultSetGroup.
    // We estimate that 1 bucket (including overhead) consumes 1KB and this cache is allowed to use up to 50% of max
    // heap space.
    long maxBucketNumber = getApproximateMaxBucketNumber(DEFAULT_HEAP_PERCENTAGE_FOR_RESULTSETGROUP_CACHE);
    LoadingCache<PinotQuery, ResultSetGroup> resultSetGroupCache = CacheBuilder.newBuilder().removalListener(listener).expireAfterAccess(1, TimeUnit.HOURS).maximumWeight(maxBucketNumber).weigher((pinotQuery, resultSetGroup) -> {
        int resultSetCount = resultSetGroup.getResultSetCount();
        int weight = 0;
        for (int idx = 0; idx < resultSetCount; ++idx) {
            com.linkedin.pinot.client.ResultSet resultSet = resultSetGroup.getResultSet(idx);
            weight += (resultSet.getColumnCount() * resultSet.getRowCount());
        }
        return weight;
    }).build(new ResultSetGroupCacheLoader(pinotThirdeyeClientConfig));
    cacheRegistry.registerResultSetGroupCache(resultSetGroupCache);
    LOGGER.info("Max bucket number for ResultSetGroup cache is set to {}", maxBucketNumber);
    // CollectionMaxDataTime Cache
    LoadingCache<String, Long> collectionMaxDataTimeCache = CacheBuilder.newBuilder().refreshAfterWrite(5, TimeUnit.MINUTES).build(new CollectionMaxDataTimeCacheLoader(resultSetGroupCache, datasetConfigDAO));
    cacheRegistry.registerCollectionMaxDataTimeCache(collectionMaxDataTimeCache);
    // Query Cache
    QueryCache queryCache = new QueryCache(thirdEyeClient, Executors.newFixedThreadPool(10));
    cacheRegistry.registerQueryCache(queryCache);
    // Dimension Filter cache
    LoadingCache<String, String> dimensionFiltersCache = CacheBuilder.newBuilder().build(new DimensionFiltersCacheLoader(cacheRegistry.getQueryCache()));
    cacheRegistry.registerDimensionFiltersCache(dimensionFiltersCache);
    // Dashboards cache
    LoadingCache<String, String> dashboardsCache = CacheBuilder.newBuilder().build(new DashboardsCacheLoader(dashboardConfigDAO));
    cacheRegistry.registerDashboardsCache(dashboardsCache);
    // Collections cache
    CollectionsCache collectionsCache = new CollectionsCache(datasetConfigDAO, config);
    cacheRegistry.registerCollectionsCache(collectionsCache);
    // DatasetConfig cache
    LoadingCache<String, DatasetConfigDTO> datasetConfigCache = CacheBuilder.newBuilder().build(new DatasetConfigCacheLoader(datasetConfigDAO));
    cacheRegistry.registerDatasetConfigCache(datasetConfigCache);
    // MetricConfig cache
    LoadingCache<MetricDataset, MetricConfigDTO> metricConfigCache = CacheBuilder.newBuilder().build(new MetricConfigCacheLoader(metricConfigDAO));
    cacheRegistry.registerMetricConfigCache(metricConfigCache);
    // DashboardConfigs cache
    LoadingCache<String, List<DashboardConfigDTO>> dashboardConfigsCache = CacheBuilder.newBuilder().build(new DashboardConfigCacheLoader(dashboardConfigDAO));
    cacheRegistry.registerDashboardConfigsCache(dashboardConfigsCache);
}

Example 4 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class AutoLoadPinotMetricsService method addNewDataset.

/**
   * Adds a new dataset to the thirdeye database
   * @param dataset
   * @param schema
   */
private void addNewDataset(String dataset, Schema schema) throws Exception {
    List<MetricFieldSpec> metricSpecs = schema.getMetricFieldSpecs();
    // Create DatasetConfig
    DatasetConfigDTO datasetConfigDTO = ConfigGenerator.generateDatasetConfig(dataset, schema);
    LOG.info("Creating dataset for {}", dataset);
    DAO_REGISTRY.getDatasetConfigDAO().save(datasetConfigDTO);
    // Create MetricConfig
    for (MetricFieldSpec metricFieldSpec : metricSpecs) {
        MetricConfigDTO metricConfigDTO = ConfigGenerator.generateMetricConfig(metricFieldSpec, dataset);
        LOG.info("Creating metric {} for {}", metricConfigDTO.getName(), dataset);
        DAO_REGISTRY.getMetricConfigDAO().save(metricConfigDTO);
    }
    // Create Default DashboardConfig
    List<Long> metricIds = ConfigGenerator.getMetricIdsFromMetricConfigs(DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset));
    DashboardConfigDTO dashboardConfigDTO = ConfigGenerator.generateDefaultDashboardConfig(dataset, metricIds);
    LOG.info("Creating default dashboard for dataset {}", dataset);
    DAO_REGISTRY.getDashboardConfigDAO().save(dashboardConfigDTO);
}

Example 5 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class AnomalyResource method getExternalDashboardUrlForMergedAnomaly.

@GET
@Path("/external-dashboard-url/{mergedAnomalyId}")
public String getExternalDashboardUrlForMergedAnomaly(@NotNull @PathParam("mergedAnomalyId") Long mergedAnomalyId) throws Exception {
    MergedAnomalyResultDTO mergedAnomalyResultDTO = mergedAnomalyResultDAO.findById(mergedAnomalyId);
    String metric = mergedAnomalyResultDTO.getMetric();
    String dataset = mergedAnomalyResultDTO.getCollection();
    Long startTime = mergedAnomalyResultDTO.getStartTime();
    Long endTime = mergedAnomalyResultDTO.getEndTime();
    MetricConfigDTO metricConfigDTO = metricConfigDAO.findByMetricAndDataset(metric, dataset);
    Map<String, String> context = new HashMap<>();
    context.put(MetricConfigBean.URL_TEMPLATE_START_TIME, String.valueOf(startTime));
    context.put(MetricConfigBean.URL_TEMPLATE_END_TIME, String.valueOf(endTime));
    StrSubstitutor strSubstitutor = new StrSubstitutor(context);
    Map<String, String> urlTemplates = metricConfigDTO.getExtSourceLinkInfo();
    for (Entry<String, String> entry : urlTemplates.entrySet()) {
        String sourceName = entry.getKey();
        String urlTemplate = entry.getValue();
        String extSourceUrl = strSubstitutor.replace(urlTemplate);
        urlTemplates.put(sourceName, extSourceUrl);
    }
    return new JSONObject(urlTemplates).toString();
}