Examples with MetricDimensionReport com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport used on opensource projects

Example 1 with MetricDimensionReport

use of com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport in project pinot by linkedin.

the class AlertTaskRunnerV2 method sendScheduledDataReport.

private void sendScheduledDataReport() throws Exception {
    AlertConfigBean.ReportConfigCollection reportConfigCollection = alertConfig.getReportConfigCollection();
    if (reportConfigCollection != null && reportConfigCollection.isEnabled()) {
        if (reportConfigCollection.getReportMetricConfigs() != null && reportConfigCollection.getReportMetricConfigs().size() > 0) {
            List<MetricDimensionReport> metricDimensionValueReports;
            // Used later to provide collection for a metric to help build the url link in report
            Map<String, MetricConfigDTO> metricMap = new HashMap<>();
            List<ContributorViewResponse> reports = new ArrayList<>();
            for (int i = 0; i < reportConfigCollection.getReportMetricConfigs().size(); i++) {
                AlertConfigBean.ReportMetricConfig reportMetricConfig = reportConfigCollection.getReportMetricConfigs().get(i);
                MetricConfigDTO metricConfig = metricConfigManager.findById(reportMetricConfig.getMetricId());
                List<String> dimensions = reportMetricConfig.getDimensions();
                if (dimensions != null && dimensions.size() > 0) {
                    for (String dimension : dimensions) {
                        ContributorViewResponse report = EmailHelper.getContributorDataForDataReport(metricConfig.getDataset(), metricConfig.getName(), Arrays.asList(dimension), reportMetricConfig.getCompareMode(), alertConfig.getReportConfigCollection().getDelayOffsetMillis(), alertConfig.getReportConfigCollection().isIntraDay());
                        if (report != null) {
                            metricMap.put(metricConfig.getName(), metricConfig);
                            reports.add(report);
                        }
                    }
                }
            }
            if (reports.size() == 0) {
                LOG.warn("Could not fetch report data for " + alertConfig.getName());
                return;
            }
            long reportStartTs = reports.get(0).getTimeBuckets().get(0).getCurrentStart();
            metricDimensionValueReports = DataReportHelper.getInstance().getDimensionReportList(reports);
            for (int i = 0; i < metricDimensionValueReports.size(); i++) {
                MetricDimensionReport report = metricDimensionValueReports.get(i);
                report.setDataset(metricMap.get(report.getMetricName()).getDataset());
                long metricId = metricMap.get(report.getMetricName()).getId();
                report.setMetricId(metricId);
                for (AlertConfigBean.ReportMetricConfig reportMetricConfig : reportConfigCollection.getReportMetricConfigs()) {
                    if (reportMetricConfig.getMetricId() == metricId) {
                        metricDimensionValueReports.get(i).setCompareMode(reportMetricConfig.getCompareMode().name());
                    }
                }
            }
            Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
            freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector/");
            freemarkerConfig.setDefaultEncoding(CHARSET);
            freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
            Map<String, Object> templateData = new HashMap<>();
            DateTimeZone timeZone = DateTimeZone.forTimeZone(DEFAULT_TIME_ZONE);
            DataReportHelper.DateFormatMethod dateFormatMethod = new DataReportHelper.DateFormatMethod(timeZone);
            templateData.put("timeZone", timeZone);
            templateData.put("dateFormat", dateFormatMethod);
            templateData.put("dashboardHost", thirdeyeConfig.getDashboardHost());
            templateData.put("fromEmail", alertConfig.getFromAddress());
            templateData.put("contactEmail", alertConfig.getReportConfigCollection().getContactEmail());
            templateData.put("reportStartDateTime", reportStartTs);
            templateData.put("metricDimensionValueReports", metricDimensionValueReports);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            try (Writer out = new OutputStreamWriter(baos, CHARSET)) {
                Template template = freemarkerConfig.getTemplate("data-report-by-metric-dimension.ftl");
                template.process(templateData, out);
                // Send email
                HtmlEmail email = new HtmlEmail();
                String alertEmailSubject = String.format("Thirdeye data report : %s", alertConfig.getName());
                String alertEmailHtml = new String(baos.toByteArray(), CHARSET);
                EmailHelper.sendEmailWithHtml(email, thirdeyeConfig.getSmtpConfiguration(), alertEmailSubject, alertEmailHtml, alertConfig.getFromAddress(), alertConfig.getRecipients());
            } catch (Exception e) {
                throw new JobExecutionException(e);
            }
        }
    }
}

Example 2 with MetricDimensionReport

use of com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport in project pinot by linkedin.

the class AlertTaskRunner method sendAlertForAnomalies.

private void sendAlertForAnomalies(String collectionAlias, List<MergedAnomalyResultDTO> results, Map<DimensionMap, List<MergedAnomalyResultDTO>> groupedResults) throws JobExecutionException {
    long anomalyStartMillis = 0;
    long anomalyEndMillis = 0;
    int anomalyResultSize = 0;
    if (CollectionUtils.isNotEmpty(results)) {
        anomalyResultSize = results.size();
        anomalyStartMillis = results.get(0).getStartTime();
        anomalyEndMillis = results.get(0).getEndTime();
        for (MergedAnomalyResultDTO mergedAnomalyResultDTO : results) {
            if (mergedAnomalyResultDTO.getStartTime() < anomalyStartMillis) {
                anomalyStartMillis = mergedAnomalyResultDTO.getStartTime();
            }
            if (mergedAnomalyResultDTO.getEndTime() > anomalyEndMillis) {
                anomalyEndMillis = mergedAnomalyResultDTO.getEndTime();
            }
        }
    }
    DateTimeZone timeZone = DateTimeZone.forTimeZone(DEFAULT_TIME_ZONE);
    DataReportHelper.DateFormatMethod dateFormatMethod = new DataReportHelper.DateFormatMethod(timeZone);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (Writer out = new OutputStreamWriter(baos, CHARSET)) {
        Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
        freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector/");
        freemarkerConfig.setDefaultEncoding(CHARSET);
        freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
        Map<String, Object> templateData = new HashMap<>();
        String metric = alertConfig.getMetric();
        String windowUnit = alertConfig.getWindowUnit().toString();
        templateData.put("groupedAnomalyResults", DataReportHelper.convertToStringKeyBasedMap(groupedResults));
        templateData.put("anomalyCount", anomalyResultSize);
        templateData.put("startTime", anomalyStartMillis);
        templateData.put("endTime", anomalyEndMillis);
        templateData.put("reportGenerationTimeMillis", System.currentTimeMillis());
        templateData.put("dateFormat", dateFormatMethod);
        templateData.put("timeZone", timeZone);
        templateData.put("collection", collectionAlias);
        templateData.put("metric", metric);
        templateData.put("windowUnit", windowUnit);
        templateData.put("dashboardHost", thirdeyeConfig.getDashboardHost());
        if (alertConfig.isReportEnabled() & alertConfig.getDimensions() != null) {
            long reportStartTs = 0;
            List<MetricDimensionReport> metricDimensionValueReports;
            List<ContributorViewResponse> reports = new ArrayList<>();
            for (String dimension : alertConfig.getDimensions()) {
                ContributorViewResponse report = EmailHelper.getContributorDataForDataReport(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension));
                if (report != null) {
                    reports.add(report);
                }
            }
            reportStartTs = reports.get(0).getTimeBuckets().get(0).getCurrentStart();
            metricDimensionValueReports = DataReportHelper.getInstance().getDimensionReportList(reports);
            templateData.put("metricDimensionValueReports", metricDimensionValueReports);
            templateData.put("reportStartDateTime", reportStartTs);
        }
        Template template = freemarkerConfig.getTemplate("anomaly-report.ftl");
        template.process(templateData, out);
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
    // Send email
    try {
        String alertEmailSubject;
        if (results.size() > 0) {
            String anomalyString = (results.size() == 1) ? "anomaly" : "anomalies";
            alertEmailSubject = String.format("Thirdeye: %s: %s - %d %s detected", alertConfig.getMetric(), collectionAlias, results.size(), anomalyString);
        } else {
            alertEmailSubject = String.format("Thirdeye data report : %s: %s", alertConfig.getMetric(), collectionAlias);
        }
        HtmlEmail email = new HtmlEmail();
        String alertEmailHtml = new String(baos.toByteArray(), CHARSET);
        EmailHelper.sendEmailWithHtml(email, thirdeyeConfig.getSmtpConfiguration(), alertEmailSubject, alertEmailHtml, alertConfig.getFromAddress(), alertConfig.getToAddresses());
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
    // once email is sent, update the last merged anomaly id as watermark in email config
    long anomalyId = 0;
    for (MergedAnomalyResultDTO anomalyResultDTO : results) {
        if (anomalyResultDTO.getId() > anomalyId) {
            anomalyId = anomalyResultDTO.getId();
        }
    }
    alertConfig.setLastNotifiedAnomalyId(anomalyId);
    emailConfigurationDAO.update(alertConfig);
    LOG.info("Sent email with {} anomalies! {}", results.size(), alertConfig);
}

Example 3 with MetricDimensionReport

use of com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport in project pinot by linkedin.

the class DataReportHelper method getDimensionReportList.

// report list metric vs groupByKey vs dimensionVal vs timeBucket vs values
public List<MetricDimensionReport> getDimensionReportList(List<ContributorViewResponse> reports) {
    List<MetricDimensionReport> ultimateResult = new ArrayList<>();
    for (ContributorViewResponse report : reports) {
        MetricDimensionReport metricDimensionReport = new MetricDimensionReport();
        String metric = report.getMetrics().get(0);
        String groupByDimension = report.getDimensions().get(0);
        metricDimensionReport.setMetricName(metric);
        metricDimensionReport.setDimensionName(groupByDimension);
        int valIndex = report.getResponseData().getSchema().getColumnsToIndexMapping().get("percentageChange");
        int dimensionIndex = report.getResponseData().getSchema().getColumnsToIndexMapping().get("dimensionValue");
        int numDimensions = report.getDimensionValuesMap().get(groupByDimension).size();
        int numBuckets = report.getTimeBuckets().size();
        // this is dimension vs timeBucketValue map, this should be sorted based on first bucket value
        Map<String, Map<String, String>> dimensionValueMap = new LinkedHashMap<>();
        // Lets populate 'OverAll' contribution
        Map<String, String> overAllValueMap = new LinkedHashMap<>();
        populateOverAllValuesMap(report, overAllValueMap);
        dimensionValueMap.put(OVER_ALL, overAllValueMap);
        Map<String, Map<String, String>> dimensionValueMapUnordered = new HashMap<>();
        for (int p = 0; p < numDimensions; p++) {
            if (p == 0) {
                metricDimensionReport.setCurrentStartTime(report.getTimeBuckets().get(0).getCurrentStart());
                metricDimensionReport.setCurrentEndTime(report.getTimeBuckets().get(numBuckets - 1).getCurrentEnd());
                metricDimensionReport.setBaselineStartTime(report.getTimeBuckets().get(0).getBaselineStart());
                metricDimensionReport.setBaselineEndTime(report.getTimeBuckets().get(numBuckets - 1).getBaselineEnd());
            }
            // valueMap is timeBucket vs value map
            LinkedHashMap<String, String> valueMap = new LinkedHashMap<>();
            String currentDimension = "";
            for (int q = 0; q < numBuckets; q++) {
                int index = p * numBuckets + q;
                currentDimension = report.getResponseData().getResponseData().get(index)[dimensionIndex];
                valueMap.put(String.valueOf(report.getTimeBuckets().get(q).getCurrentStart()), String.format(DECIMAL_FORMATTER, Double.valueOf(report.getResponseData().getResponseData().get(index)[valIndex])));
            }
            dimensionValueMapUnordered.put(currentDimension, valueMap);
        }
        orderDimensionValueMap(dimensionValueMapUnordered, dimensionValueMap, report.getDimensionValuesMap().get(groupByDimension));
        metricDimensionReport.setSubDimensionValueMap(dimensionValueMap);
        populateShareAndTotalMap(report, metricDimensionReport, metric, groupByDimension);
        ultimateResult.add(metricDimensionReport);
    }
    return ultimateResult;
}