Examples with ContributorViewResponse com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse used on opensource projects

Example 1 with ContributorViewResponse

use of com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse in project pinot by linkedin.

the class TimeSeriesResource method getContributorDataForDimension.

private TimeSeriesCompareMetricView getContributorDataForDimension(long metricId, long currentStart, long currentEnd, long baselineStart, long baselineEnd, String dimension, String filters, String granularity) {
    MetricConfigDTO metricConfigDTO = metricConfigDAO.findById(metricId);
    TimeSeriesCompareMetricView timeSeriesCompareMetricView = new TimeSeriesCompareMetricView(metricConfigDTO.getName(), metricId, currentStart, currentEnd);
    try {
        String dataset = metricConfigDTO.getDataset();
        ContributorViewRequest request = new ContributorViewRequest();
        request.setCollection(dataset);
        MetricExpression metricExpression = ThirdEyeUtils.getMetricExpressionFromMetricConfig(metricConfigDTO);
        request.setMetricExpressions(Arrays.asList(metricExpression));
        DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(dataset);
        request.setBaselineStart(new DateTime(baselineStart, timeZoneForCollection));
        request.setBaselineEnd(new DateTime(baselineEnd, timeZoneForCollection));
        request.setCurrentStart(new DateTime(currentStart, timeZoneForCollection));
        request.setCurrentEnd(new DateTime(currentEnd, timeZoneForCollection));
        request.setTimeGranularity(Utils.getAggregationTimeGranularity(granularity, dataset));
        if (filters != null && !filters.isEmpty()) {
            filters = URLDecoder.decode(filters, "UTF-8");
            request.setFilters(ThirdEyeUtils.convertToMultiMap(filters));
        }
        request.setGroupByDimensions(Arrays.asList(dimension));
        ContributorViewHandler handler = new ContributorViewHandler(queryCache);
        ContributorViewResponse response = handler.process(request);
        // Assign the time buckets
        List<Long> timeBucketsCurrent = new ArrayList<>();
        List<Long> timeBucketsBaseline = new ArrayList<>();
        timeSeriesCompareMetricView.setTimeBucketsCurrent(timeBucketsCurrent);
        timeSeriesCompareMetricView.setTimeBucketsBaseline(timeBucketsBaseline);
        Map<String, ValuesContainer> subDimensionValuesMap = new LinkedHashMap<>();
        timeSeriesCompareMetricView.setSubDimensionContributionMap(subDimensionValuesMap);
        int timeBuckets = response.getTimeBuckets().size();
        // this is for over all values
        ValuesContainer vw = new ValuesContainer();
        subDimensionValuesMap.put(ALL, vw);
        vw.setCurrentValues(new double[timeBuckets]);
        vw.setBaselineValues(new double[timeBuckets]);
        vw.setPercentageChange(new String[timeBuckets]);
        vw.setCumulativeCurrentValues(new double[timeBuckets]);
        vw.setCumulativeBaselineValues(new double[timeBuckets]);
        vw.setCumulativePercentageChange(new String[timeBuckets]);
        // lets find the indices
        int subDimensionIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("dimensionValue");
        int currentValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("currentValue");
        int baselineValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("baselineValue");
        int percentageChangeIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("percentageChange");
        int cumCurrentValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativeCurrentValue");
        int cumBaselineValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativeBaselineValue");
        int cumPercentageChangeIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativePercentageChange");
        // populate current and baseline time buckets
        for (int i = 0; i < timeBuckets; i++) {
            TimeBucket tb = response.getTimeBuckets().get(i);
            timeBucketsCurrent.add(tb.getCurrentStart());
            timeBucketsBaseline.add(tb.getBaselineStart());
        }
        // set current and baseline values for sub dimensions
        for (int i = 0; i < response.getResponseData().getResponseData().size(); i++) {
            String[] data = response.getResponseData().getResponseData().get(i);
            String subDimension = data[subDimensionIndex];
            Double currentVal = Double.valueOf(data[currentValueIndex]);
            Double baselineVal = Double.valueOf(data[baselineValueIndex]);
            Double percentageChangeVal = Double.valueOf(data[percentageChangeIndex]);
            Double cumCurrentVal = Double.valueOf(data[cumCurrentValueIndex]);
            Double cumBaselineVal = Double.valueOf(data[cumBaselineValueIndex]);
            Double cumPercentageChangeVal = Double.valueOf(data[cumPercentageChangeIndex]);
            int index = i % timeBuckets;
            // set overAll values
            vw.getCurrentValues()[index] += currentVal;
            vw.getBaselineValues()[index] += baselineVal;
            vw.getCumulativeCurrentValues()[index] += cumCurrentVal;
            vw.getCumulativeBaselineValues()[index] += cumBaselineVal;
            // set individual sub-dimension values
            if (!subDimensionValuesMap.containsKey(subDimension)) {
                ValuesContainer subDimVals = new ValuesContainer();
                subDimVals.setCurrentValues(new double[timeBuckets]);
                subDimVals.setBaselineValues(new double[timeBuckets]);
                subDimVals.setPercentageChange(new String[timeBuckets]);
                subDimVals.setCumulativeCurrentValues(new double[timeBuckets]);
                subDimVals.setCumulativeBaselineValues(new double[timeBuckets]);
                subDimVals.setCumulativePercentageChange(new String[timeBuckets]);
                subDimensionValuesMap.put(subDimension, subDimVals);
            }
            subDimensionValuesMap.get(subDimension).getCurrentValues()[index] = currentVal;
            subDimensionValuesMap.get(subDimension).getBaselineValues()[index] = baselineVal;
            subDimensionValuesMap.get(subDimension).getPercentageChange()[index] = String.format(DECIMAL_FORMAT, percentageChangeVal);
            subDimensionValuesMap.get(subDimension).getCumulativeCurrentValues()[index] = cumCurrentVal;
            subDimensionValuesMap.get(subDimension).getCumulativeBaselineValues()[index] = cumBaselineVal;
            subDimensionValuesMap.get(subDimension).getCumulativePercentageChange()[index] = String.format(DECIMAL_FORMAT, cumPercentageChangeVal);
        }
        // TODO : compute cumulative values for all
        for (int i = 0; i < vw.getCurrentValues().length; i++) {
            vw.getPercentageChange()[i] = String.format(DECIMAL_FORMAT, getPercentageChange(vw.getCurrentValues()[i], vw.getBaselineValues()[i]));
            vw.getCumulativePercentageChange()[i] = String.format(DECIMAL_FORMAT, getPercentageChange(vw.getCumulativeCurrentValues()[i], vw.getCumulativeBaselineValues()[i]));
        }
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new WebApplicationException(e);
    }
    return timeSeriesCompareMetricView;
}

Example 2 with ContributorViewResponse

use of com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse in project pinot by linkedin.

the class ContributorTest method main.

public static void main(String[] args) throws Exception {
    ContributorViewRequest request = new ContributorViewRequest();
    String collection = "thirdeyeAbook";
    DateTime baselineStart = new DateTime(2016, 3, 23, 00, 00);
    List<MetricExpression> metricExpressions = new ArrayList<>();
    metricExpressions.add(new MetricExpression("__COUNT", "__COUNT"));
    request.setCollection(collection);
    request.setBaselineStart(baselineStart);
    request.setBaselineEnd(baselineStart.plusDays(1));
    request.setCurrentStart(baselineStart.plusDays(7));
    request.setCurrentEnd(baselineStart.plusDays(8));
    request.setTimeGranularity(new TimeGranularity(1, TimeUnit.HOURS));
    request.setMetricExpressions(metricExpressions);
    // TODO
    PinotThirdEyeClient pinotThirdEyeClient = PinotThirdEyeClient.getDefaultTestClient();
    // make
    // this
    // configurable;
    QueryCache queryCache = new QueryCache(pinotThirdEyeClient, Executors.newFixedThreadPool(10));
    ContributorViewHandler handler = new ContributorViewHandler(queryCache);
    ContributorViewResponse response = handler.process(request);
    ObjectMapper mapper = new ObjectMapper();
    String jsonResponse = mapper.writeValueAsString(response);
    System.out.println(jsonResponse);
}

Example 3 with ContributorViewResponse

use of com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse in project pinot by linkedin.

the class AlertTaskRunnerV2 method sendScheduledDataReport.

private void sendScheduledDataReport() throws Exception {
    AlertConfigBean.ReportConfigCollection reportConfigCollection = alertConfig.getReportConfigCollection();
    if (reportConfigCollection != null && reportConfigCollection.isEnabled()) {
        if (reportConfigCollection.getReportMetricConfigs() != null && reportConfigCollection.getReportMetricConfigs().size() > 0) {
            List<MetricDimensionReport> metricDimensionValueReports;
            // Used later to provide collection for a metric to help build the url link in report
            Map<String, MetricConfigDTO> metricMap = new HashMap<>();
            List<ContributorViewResponse> reports = new ArrayList<>();
            for (int i = 0; i < reportConfigCollection.getReportMetricConfigs().size(); i++) {
                AlertConfigBean.ReportMetricConfig reportMetricConfig = reportConfigCollection.getReportMetricConfigs().get(i);
                MetricConfigDTO metricConfig = metricConfigManager.findById(reportMetricConfig.getMetricId());
                List<String> dimensions = reportMetricConfig.getDimensions();
                if (dimensions != null && dimensions.size() > 0) {
                    for (String dimension : dimensions) {
                        ContributorViewResponse report = EmailHelper.getContributorDataForDataReport(metricConfig.getDataset(), metricConfig.getName(), Arrays.asList(dimension), reportMetricConfig.getCompareMode(), alertConfig.getReportConfigCollection().getDelayOffsetMillis(), alertConfig.getReportConfigCollection().isIntraDay());
                        if (report != null) {
                            metricMap.put(metricConfig.getName(), metricConfig);
                            reports.add(report);
                        }
                    }
                }
            }
            if (reports.size() == 0) {
                LOG.warn("Could not fetch report data for " + alertConfig.getName());
                return;
            }
            long reportStartTs = reports.get(0).getTimeBuckets().get(0).getCurrentStart();
            metricDimensionValueReports = DataReportHelper.getInstance().getDimensionReportList(reports);
            for (int i = 0; i < metricDimensionValueReports.size(); i++) {
                MetricDimensionReport report = metricDimensionValueReports.get(i);
                report.setDataset(metricMap.get(report.getMetricName()).getDataset());
                long metricId = metricMap.get(report.getMetricName()).getId();
                report.setMetricId(metricId);
                for (AlertConfigBean.ReportMetricConfig reportMetricConfig : reportConfigCollection.getReportMetricConfigs()) {
                    if (reportMetricConfig.getMetricId() == metricId) {
                        metricDimensionValueReports.get(i).setCompareMode(reportMetricConfig.getCompareMode().name());
                    }
                }
            }
            Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
            freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector/");
            freemarkerConfig.setDefaultEncoding(CHARSET);
            freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
            Map<String, Object> templateData = new HashMap<>();
            DateTimeZone timeZone = DateTimeZone.forTimeZone(DEFAULT_TIME_ZONE);
            DataReportHelper.DateFormatMethod dateFormatMethod = new DataReportHelper.DateFormatMethod(timeZone);
            templateData.put("timeZone", timeZone);
            templateData.put("dateFormat", dateFormatMethod);
            templateData.put("dashboardHost", thirdeyeConfig.getDashboardHost());
            templateData.put("fromEmail", alertConfig.getFromAddress());
            templateData.put("contactEmail", alertConfig.getReportConfigCollection().getContactEmail());
            templateData.put("reportStartDateTime", reportStartTs);
            templateData.put("metricDimensionValueReports", metricDimensionValueReports);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            try (Writer out = new OutputStreamWriter(baos, CHARSET)) {
                Template template = freemarkerConfig.getTemplate("data-report-by-metric-dimension.ftl");
                template.process(templateData, out);
                // Send email
                HtmlEmail email = new HtmlEmail();
                String alertEmailSubject = String.format("Thirdeye data report : %s", alertConfig.getName());
                String alertEmailHtml = new String(baos.toByteArray(), CHARSET);
                EmailHelper.sendEmailWithHtml(email, thirdeyeConfig.getSmtpConfiguration(), alertEmailSubject, alertEmailHtml, alertConfig.getFromAddress(), alertConfig.getRecipients());
            } catch (Exception e) {
                throw new JobExecutionException(e);
            }
        }
    }
}

Example 4 with ContributorViewResponse

use of com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse in project pinot by linkedin.

the class AlertTaskRunner method sendAlertForAnomalies.

private void sendAlertForAnomalies(String collectionAlias, List<MergedAnomalyResultDTO> results, Map<DimensionMap, List<MergedAnomalyResultDTO>> groupedResults) throws JobExecutionException {
    long anomalyStartMillis = 0;
    long anomalyEndMillis = 0;
    int anomalyResultSize = 0;
    if (CollectionUtils.isNotEmpty(results)) {
        anomalyResultSize = results.size();
        anomalyStartMillis = results.get(0).getStartTime();
        anomalyEndMillis = results.get(0).getEndTime();
        for (MergedAnomalyResultDTO mergedAnomalyResultDTO : results) {
            if (mergedAnomalyResultDTO.getStartTime() < anomalyStartMillis) {
                anomalyStartMillis = mergedAnomalyResultDTO.getStartTime();
            }
            if (mergedAnomalyResultDTO.getEndTime() > anomalyEndMillis) {
                anomalyEndMillis = mergedAnomalyResultDTO.getEndTime();
            }
        }
    }
    DateTimeZone timeZone = DateTimeZone.forTimeZone(DEFAULT_TIME_ZONE);
    DataReportHelper.DateFormatMethod dateFormatMethod = new DataReportHelper.DateFormatMethod(timeZone);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (Writer out = new OutputStreamWriter(baos, CHARSET)) {
        Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
        freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector/");
        freemarkerConfig.setDefaultEncoding(CHARSET);
        freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
        Map<String, Object> templateData = new HashMap<>();
        String metric = alertConfig.getMetric();
        String windowUnit = alertConfig.getWindowUnit().toString();
        templateData.put("groupedAnomalyResults", DataReportHelper.convertToStringKeyBasedMap(groupedResults));
        templateData.put("anomalyCount", anomalyResultSize);
        templateData.put("startTime", anomalyStartMillis);
        templateData.put("endTime", anomalyEndMillis);
        templateData.put("reportGenerationTimeMillis", System.currentTimeMillis());
        templateData.put("dateFormat", dateFormatMethod);
        templateData.put("timeZone", timeZone);
        templateData.put("collection", collectionAlias);
        templateData.put("metric", metric);
        templateData.put("windowUnit", windowUnit);
        templateData.put("dashboardHost", thirdeyeConfig.getDashboardHost());
        if (alertConfig.isReportEnabled() & alertConfig.getDimensions() != null) {
            long reportStartTs = 0;
            List<MetricDimensionReport> metricDimensionValueReports;
            List<ContributorViewResponse> reports = new ArrayList<>();
            for (String dimension : alertConfig.getDimensions()) {
                ContributorViewResponse report = EmailHelper.getContributorDataForDataReport(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension));
                if (report != null) {
                    reports.add(report);
                }
            }
            reportStartTs = reports.get(0).getTimeBuckets().get(0).getCurrentStart();
            metricDimensionValueReports = DataReportHelper.getInstance().getDimensionReportList(reports);
            templateData.put("metricDimensionValueReports", metricDimensionValueReports);
            templateData.put("reportStartDateTime", reportStartTs);
        }
        Template template = freemarkerConfig.getTemplate("anomaly-report.ftl");
        template.process(templateData, out);
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
    // Send email
    try {
        String alertEmailSubject;
        if (results.size() > 0) {
            String anomalyString = (results.size() == 1) ? "anomaly" : "anomalies";
            alertEmailSubject = String.format("Thirdeye: %s: %s - %d %s detected", alertConfig.getMetric(), collectionAlias, results.size(), anomalyString);
        } else {
            alertEmailSubject = String.format("Thirdeye data report : %s: %s", alertConfig.getMetric(), collectionAlias);
        }
        HtmlEmail email = new HtmlEmail();
        String alertEmailHtml = new String(baos.toByteArray(), CHARSET);
        EmailHelper.sendEmailWithHtml(email, thirdeyeConfig.getSmtpConfiguration(), alertEmailSubject, alertEmailHtml, alertConfig.getFromAddress(), alertConfig.getToAddresses());
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
    // once email is sent, update the last merged anomaly id as watermark in email config
    long anomalyId = 0;
    for (MergedAnomalyResultDTO anomalyResultDTO : results) {
        if (anomalyResultDTO.getId() > anomalyId) {
            anomalyId = anomalyResultDTO.getId();
        }
    }
    alertConfig.setLastNotifiedAnomalyId(anomalyId);
    emailConfigurationDAO.update(alertConfig);
    LOG.info("Sent email with {} anomalies! {}", results.size(), alertConfig);
}

Example 5 with ContributorViewResponse

use of com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse in project pinot by linkedin.

the class DataReportHelper method getDimensionReportList.

// report list metric vs groupByKey vs dimensionVal vs timeBucket vs values
public List<MetricDimensionReport> getDimensionReportList(List<ContributorViewResponse> reports) {
    List<MetricDimensionReport> ultimateResult = new ArrayList<>();
    for (ContributorViewResponse report : reports) {
        MetricDimensionReport metricDimensionReport = new MetricDimensionReport();
        String metric = report.getMetrics().get(0);
        String groupByDimension = report.getDimensions().get(0);
        metricDimensionReport.setMetricName(metric);
        metricDimensionReport.setDimensionName(groupByDimension);
        int valIndex = report.getResponseData().getSchema().getColumnsToIndexMapping().get("percentageChange");
        int dimensionIndex = report.getResponseData().getSchema().getColumnsToIndexMapping().get("dimensionValue");
        int numDimensions = report.getDimensionValuesMap().get(groupByDimension).size();
        int numBuckets = report.getTimeBuckets().size();
        // this is dimension vs timeBucketValue map, this should be sorted based on first bucket value
        Map<String, Map<String, String>> dimensionValueMap = new LinkedHashMap<>();
        // Lets populate 'OverAll' contribution
        Map<String, String> overAllValueMap = new LinkedHashMap<>();
        populateOverAllValuesMap(report, overAllValueMap);
        dimensionValueMap.put(OVER_ALL, overAllValueMap);
        Map<String, Map<String, String>> dimensionValueMapUnordered = new HashMap<>();
        for (int p = 0; p < numDimensions; p++) {
            if (p == 0) {
                metricDimensionReport.setCurrentStartTime(report.getTimeBuckets().get(0).getCurrentStart());
                metricDimensionReport.setCurrentEndTime(report.getTimeBuckets().get(numBuckets - 1).getCurrentEnd());
                metricDimensionReport.setBaselineStartTime(report.getTimeBuckets().get(0).getBaselineStart());
                metricDimensionReport.setBaselineEndTime(report.getTimeBuckets().get(numBuckets - 1).getBaselineEnd());
            }
            // valueMap is timeBucket vs value map
            LinkedHashMap<String, String> valueMap = new LinkedHashMap<>();
            String currentDimension = "";
            for (int q = 0; q < numBuckets; q++) {
                int index = p * numBuckets + q;
                currentDimension = report.getResponseData().getResponseData().get(index)[dimensionIndex];
                valueMap.put(String.valueOf(report.getTimeBuckets().get(q).getCurrentStart()), String.format(DECIMAL_FORMATTER, Double.valueOf(report.getResponseData().getResponseData().get(index)[valIndex])));
            }
            dimensionValueMapUnordered.put(currentDimension, valueMap);
        }
        orderDimensionValueMap(dimensionValueMapUnordered, dimensionValueMap, report.getDimensionValuesMap().get(groupByDimension));
        metricDimensionReport.setSubDimensionValueMap(dimensionValueMap);
        populateShareAndTotalMap(report, metricDimensionReport, metric, groupByDimension);
        ultimateResult.add(metricDimensionReport);
    }
    return ultimateResult;
}