Examples with MetricConfigDTO com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO used on opensource projects

Example 31 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class AutoLoadPinotMetricsService method checkMetricAsDimensionDataset.

private void checkMetricAsDimensionDataset(DatasetConfigDTO datasetConfigDTO, Schema schema) {
    String dataset = datasetConfigDTO.getDataset();
    String metricNamesColumn = datasetConfigDTO.getMetricNamesColumn();
    String metricValuesColumn = datasetConfigDTO.getMetricValuesColumn();
    FieldSpec metricValuesColumnFieldSpec = schema.getFieldSpecFor(metricValuesColumn);
    String dashboardName = ThirdEyeUtils.getDefaultDashboardName(dataset);
    // remove metricNamesColumn from dimensions if exists
    List<String> dimensions = datasetConfigDTO.getDimensions();
    if (dimensions.contains(metricNamesColumn)) {
        dimensions.removeAll(Lists.newArrayList(metricNamesColumn));
        datasetConfigDTO.setDimensions(dimensions);
        DAO_REGISTRY.getDatasetConfigDAO().update(datasetConfigDTO);
    }
    // remove metricValuesColumn from metrics if exists
    MetricConfigDTO metricConfigDTO = DAO_REGISTRY.getMetricConfigDAO().findByMetricAndDataset(metricValuesColumn, dataset);
    if (metricConfigDTO != null) {
        Long metricId = metricConfigDTO.getId();
        DAO_REGISTRY.getMetricConfigDAO().delete(metricConfigDTO);
        // remove metricValuesColumn id from default dashboard
        DashboardConfigDTO dashboardConfig = DAO_REGISTRY.getDashboardConfigDAO().findByName(dashboardName);
        List<Long> dashboardMetricIds = dashboardConfig.getMetricIds();
        dashboardMetricIds.removeAll(Lists.newArrayList(metricId));
        LOG.info("Updating dashboard config for {}", dashboardName);
        DAO_REGISTRY.getDashboardConfigDAO().update(dashboardConfig);
    }
    if (datasetConfigDTO.isAutoDiscoverMetrics()) {
        // query pinot to fetch distinct metricNamesColumn
        List<String> allDistinctMetricNames = fetchMetricAsADimensionMetrics(dataset, metricNamesColumn);
        // create metrics for these metric names, if they dont exist
        List<MetricConfigDTO> existingMetricConfigs = DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset);
        List<String> existingMetricNames = Lists.newArrayList();
        for (MetricConfigDTO existingMetricConfig : existingMetricConfigs) {
            existingMetricNames.add(existingMetricConfig.getName());
        }
        allDistinctMetricNames.removeAll(existingMetricNames);
        for (String metricName : allDistinctMetricNames) {
            LOG.info("Creating metric config for {}", metricName);
            MetricFieldSpec metricFieldSpec = new MetricFieldSpec(metricName, metricValuesColumnFieldSpec.getDataType());
            MetricConfigDTO metricConfig = ConfigGenerator.generateMetricConfig(metricFieldSpec, dataset);
            DAO_REGISTRY.getMetricConfigDAO().save(metricConfig);
        }
        // Add metrics to default dashboard
        List<Long> allMetricIds = ConfigGenerator.getMetricIdsFromMetricConfigs(DAO_REGISTRY.getMetricConfigDAO().findByDataset(dataset));
        DashboardConfigDTO dashboardConfig = DAO_REGISTRY.getDashboardConfigDAO().findByName(dashboardName);
        dashboardConfig.setMetricIds(allMetricIds);
        LOG.info("Creating dashboard config for {}", dashboardName);
        DAO_REGISTRY.getDashboardConfigDAO().update(dashboardConfig);
    }
}

Example 32 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class AlertTaskRunnerV2 method sendScheduledDataReport.

private void sendScheduledDataReport() throws Exception {
    AlertConfigBean.ReportConfigCollection reportConfigCollection = alertConfig.getReportConfigCollection();
    if (reportConfigCollection != null && reportConfigCollection.isEnabled()) {
        if (reportConfigCollection.getReportMetricConfigs() != null && reportConfigCollection.getReportMetricConfigs().size() > 0) {
            List<MetricDimensionReport> metricDimensionValueReports;
            // Used later to provide collection for a metric to help build the url link in report
            Map<String, MetricConfigDTO> metricMap = new HashMap<>();
            List<ContributorViewResponse> reports = new ArrayList<>();
            for (int i = 0; i < reportConfigCollection.getReportMetricConfigs().size(); i++) {
                AlertConfigBean.ReportMetricConfig reportMetricConfig = reportConfigCollection.getReportMetricConfigs().get(i);
                MetricConfigDTO metricConfig = metricConfigManager.findById(reportMetricConfig.getMetricId());
                List<String> dimensions = reportMetricConfig.getDimensions();
                if (dimensions != null && dimensions.size() > 0) {
                    for (String dimension : dimensions) {
                        ContributorViewResponse report = EmailHelper.getContributorDataForDataReport(metricConfig.getDataset(), metricConfig.getName(), Arrays.asList(dimension), reportMetricConfig.getCompareMode(), alertConfig.getReportConfigCollection().getDelayOffsetMillis(), alertConfig.getReportConfigCollection().isIntraDay());
                        if (report != null) {
                            metricMap.put(metricConfig.getName(), metricConfig);
                            reports.add(report);
                        }
                    }
                }
            }
            if (reports.size() == 0) {
                LOG.warn("Could not fetch report data for " + alertConfig.getName());
                return;
            }
            long reportStartTs = reports.get(0).getTimeBuckets().get(0).getCurrentStart();
            metricDimensionValueReports = DataReportHelper.getInstance().getDimensionReportList(reports);
            for (int i = 0; i < metricDimensionValueReports.size(); i++) {
                MetricDimensionReport report = metricDimensionValueReports.get(i);
                report.setDataset(metricMap.get(report.getMetricName()).getDataset());
                long metricId = metricMap.get(report.getMetricName()).getId();
                report.setMetricId(metricId);
                for (AlertConfigBean.ReportMetricConfig reportMetricConfig : reportConfigCollection.getReportMetricConfigs()) {
                    if (reportMetricConfig.getMetricId() == metricId) {
                        metricDimensionValueReports.get(i).setCompareMode(reportMetricConfig.getCompareMode().name());
                    }
                }
            }
            Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
            freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector/");
            freemarkerConfig.setDefaultEncoding(CHARSET);
            freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
            Map<String, Object> templateData = new HashMap<>();
            DateTimeZone timeZone = DateTimeZone.forTimeZone(DEFAULT_TIME_ZONE);
            DataReportHelper.DateFormatMethod dateFormatMethod = new DataReportHelper.DateFormatMethod(timeZone);
            templateData.put("timeZone", timeZone);
            templateData.put("dateFormat", dateFormatMethod);
            templateData.put("dashboardHost", thirdeyeConfig.getDashboardHost());
            templateData.put("fromEmail", alertConfig.getFromAddress());
            templateData.put("contactEmail", alertConfig.getReportConfigCollection().getContactEmail());
            templateData.put("reportStartDateTime", reportStartTs);
            templateData.put("metricDimensionValueReports", metricDimensionValueReports);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            try (Writer out = new OutputStreamWriter(baos, CHARSET)) {
                Template template = freemarkerConfig.getTemplate("data-report-by-metric-dimension.ftl");
                template.process(templateData, out);
                // Send email
                HtmlEmail email = new HtmlEmail();
                String alertEmailSubject = String.format("Thirdeye data report : %s", alertConfig.getName());
                String alertEmailHtml = new String(baos.toByteArray(), CHARSET);
                EmailHelper.sendEmailWithHtml(email, thirdeyeConfig.getSmtpConfiguration(), alertEmailSubject, alertEmailHtml, alertConfig.getFromAddress(), alertConfig.getRecipients());
            } catch (Exception e) {
                throw new JobExecutionException(e);
            }
        }
    }
}

Example 33 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class PqlUtils method getSelectionClause.

/**
   * SUM each metric over the current time bucket.
   */
static String getSelectionClause(List<MetricFunction> metricFunctions) {
    StringBuilder builder = new StringBuilder();
    String delim = "";
    for (MetricFunction function : metricFunctions) {
        builder.append(delim);
        String metricName = null;
        if (function.getMetricName().equals("*")) {
            metricName = "*";
        } else {
            String metricId = function.getMetricName().replaceAll(MetricConfigBean.DERIVED_METRIC_ID_PREFIX, "");
            MetricConfigDTO metricConfig = DAO_REGISTRY.getMetricConfigDAO().findById(Long.valueOf(metricId));
            metricName = metricConfig.getName();
        }
        builder.append(function.getFunctionName()).append("(").append(metricName).append(")");
        delim = ", ";
    }
    return builder.toString();
}

Example 34 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class PqlUtils method getMetricWhereClause.

static String getMetricWhereClause(MetricFunction metricFunction, String metricNameColumn) {
    StringBuilder builder = new StringBuilder();
    if (!metricFunction.getMetricName().equals("*")) {
        builder.append(" AND ");
        String metricId = metricFunction.getMetricName().replaceAll(MetricConfigBean.DERIVED_METRIC_ID_PREFIX, "");
        MetricConfigDTO metricConfig = DAO_REGISTRY.getMetricConfigDAO().findById(Long.valueOf(metricId));
        String metricName = metricConfig.getName();
        builder.append(String.format("%s='%s'", metricNameColumn, metricName));
    }
    return builder.toString();
}

Example 35 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class CacheResource method refreshMetricConfigCache.

@POST
@Path("/refresh/metricConfig")
public Response refreshMetricConfigCache() {
    LoadingCache<MetricDataset, MetricConfigDTO> cache = CACHE_INSTANCE.getMetricConfigCache();
    List<String> collections = CACHE_INSTANCE.getCollectionsCache().getCollections();
    for (String collection : collections) {
        List<MetricConfigDTO> metricConfigs = DAO_REGISTRY.getMetricConfigDAO().findByDataset(collection);
        for (MetricConfigDTO metricConfig : metricConfigs) {
            cache.refresh(new MetricDataset(metricConfig.getName(), metricConfig.getDataset()));
        }
    }
    return Response.ok().build();
}