Examples with MergedAnomalyResultDTO com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO used on opensource projects

Example 36 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class FetchMetricDataAndExistingAnomaliesTool method fetchMergedAnomaliesInRangeByFunctionId.

public List<ResultNode> fetchMergedAnomaliesInRangeByFunctionId(long functionId, DateTime startTime, DateTime endTime) {
    AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
    LOG.info("Loading merged anaomaly results of functionId {} from db...", functionId);
    List<ResultNode> resultNodes = new ArrayList<>();
    if (anomalyFunction == null) {
        // no such function
        return resultNodes;
    }
    List<MergedAnomalyResultDTO> mergedResults = mergedAnomalyResultDAO.findByStartTimeInRangeAndFunctionId(startTime.getMillis(), endTime.getMillis(), functionId);
    for (MergedAnomalyResultDTO mergedResult : mergedResults) {
        ResultNode res = new ResultNode();
        res.functionId = functionId;
        res.functionName = anomalyFunction.getFunctionName();
        res.startTime = new DateTime(mergedResult.getStartTime());
        res.endTime = new DateTime(mergedResult.getEndTime());
        res.dimensions = mergedResult.getDimensions();
        res.setFilters(anomalyFunction.getFilters());
        res.severity = mergedResult.getWeight();
        AnomalyFeedbackDTO feedback = mergedResult.getFeedback();
        res.feedbackType = (feedback == null) ? null : feedback.getFeedbackType();
        resultNodes.add(res);
    }
    return resultNodes;
}

Example 37 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class GenerateAnomalyReport method buildReport.

void buildReport() {
    List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
    for (String collection : collections) {
        anomalies.addAll(anomalyResultManager.findByCollectionTime(collection, startTime.getTime(), endTime.getTime(), false));
    }
    if (anomalies.size() == 0) {
        System.out.println("No anomalies found, please check the report config... exiting");
    } else {
        Set<String> metrics = new HashSet<>();
        int alertedAnomalies = 0;
        int feedbackCollected = 0;
        int trueAlert = 0;
        int falseAlert = 0;
        int nonActionable = 0;
        List<AnomalyReportDTO> anomalyReportDTOList = new ArrayList<>();
        for (MergedAnomalyResultDTO anomaly : anomalies) {
            metrics.add(anomaly.getMetric());
            if (anomaly.getFeedback() != null) {
                feedbackCollected++;
                if (anomaly.getFeedback().getFeedbackType().equals(AnomalyFeedbackType.ANOMALY)) {
                    trueAlert++;
                } else if (anomaly.getFeedback().getFeedbackType().equals(AnomalyFeedbackType.NOT_ANOMALY)) {
                    falseAlert++;
                } else {
                    nonActionable++;
                }
            }
            String feedbackVal = getFeedback(anomaly.getFeedback() == null ? "NA" : anomaly.getFeedback().getFeedbackType().name());
            AnomalyReportDTO anomalyReportDTO = new AnomalyReportDTO(String.valueOf(anomaly.getId()), feedbackVal, String.format("%+.2f", anomaly.getWeight()), anomaly.getMetric(), new Date(anomaly.getStartTime()).toString(), String.format("%.2f", getTimeDiffInHours(anomaly.getStartTime(), anomaly.getEndTime())), getAnomalyURL(anomaly));
            // include notified alerts only in the email
            if (anomaly.isNotified()) {
                alertedAnomalies++;
                anomalyReportDTOList.add(anomalyReportDTO);
            }
        }
        Map<String, Object> templateData = new HashMap<>();
        templateData.put("startTime", startTime.toString());
        templateData.put("endTime", endTime.toString());
        templateData.put("anomalyCount", anomalies.size());
        templateData.put("metricsCount", metrics.size());
        templateData.put("notifiedCount", alertedAnomalies);
        templateData.put("feedbackCount", feedbackCollected);
        templateData.put("trueAlertCount", trueAlert);
        templateData.put("falseAlertCount", falseAlert);
        templateData.put("nonActionableCount", nonActionable);
        templateData.put("datasets", String.join(", ", collections));
        templateData.put("anomalyDetails", anomalyReportDTOList);
        buildEmailTemplateAndSendAlert(templateData);
    }
}

Example 38 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class AnomaliesResource method getAnomalyDetails.

/**
   * Generates Anomaly Details for each merged anomaly
   * @param mergedAnomaly
   * @param datasetConfig
   * @param timeSeriesDateFormatter
   * @param startEndDateFormatterHours
   * @param startEndDateFormatterDays
   * @param externalUrl
   * @return
   */
private AnomalyDetails getAnomalyDetails(MergedAnomalyResultDTO mergedAnomaly, DatasetConfigDTO datasetConfig, DateTimeFormatter timeSeriesDateFormatter, DateTimeFormatter startEndDateFormatterHours, DateTimeFormatter startEndDateFormatterDays, String externalUrl) throws Exception {
    String dataset = datasetConfig.getDataset();
    String metricName = mergedAnomaly.getMetric();
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyFunctionDAO.findById(mergedAnomaly.getFunctionId());
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    String aggGranularity = constructAggGranularity(datasetConfig);
    long anomalyStartTime = mergedAnomaly.getStartTime();
    long anomalyEndTime = mergedAnomaly.getEndTime();
    TimeRange range = getTimeseriesOffsetedTimes(anomalyStartTime, anomalyEndTime, datasetConfig);
    long currentStartTime = range.getStart();
    long currentEndTime = range.getEnd();
    DimensionMap dimensions = mergedAnomaly.getDimensions();
    TimeGranularity timeGranularity = Utils.getAggregationTimeGranularity(aggGranularity, anomalyFunctionSpec.getCollection());
    long bucketMillis = timeGranularity.toMillis();
    AnomalyDetails anomalyDetails = null;
    try {
        AnomalyDetectionInputContext adInputContext = TimeBasedAnomalyMerger.fetchDataByDimension(currentStartTime, currentEndTime, dimensions, anomalyFunction, mergedAnomalyResultDAO, overrideConfigDAO, true);
        MetricTimeSeries metricTimeSeries = adInputContext.getDimensionKeyMetricTimeSeriesMap().get(dimensions);
        // Transform time series with scaling factor
        List<ScalingFactor> scalingFactors = adInputContext.getScalingFactors();
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, currentStartTime, scalingFactors, anomalyFunctionSpec.getTopicMetric(), properties);
        }
        List<MergedAnomalyResultDTO> knownAnomalies = adInputContext.getKnownMergedAnomalies().get(dimensions);
        // Known anomalies are ignored (the null parameter) because 1. we can reduce users' waiting time and 2. presentation
        // data does not need to be as accurate as the one used for detecting anomalies
        AnomalyTimelinesView anomalyTimelinesView = anomalyFunction.getTimeSeriesView(metricTimeSeries, bucketMillis, anomalyFunctionSpec.getTopicMetric(), currentStartTime, currentEndTime, knownAnomalies);
        anomalyDetails = constructAnomalyDetails(metricName, dataset, datasetConfig, mergedAnomaly, anomalyFunctionSpec, currentStartTime, currentEndTime, anomalyTimelinesView, timeSeriesDateFormatter, startEndDateFormatterHours, startEndDateFormatterDays, externalUrl);
    } catch (Exception e) {
        LOG.error("Exception in constructing anomaly wrapper for anomaly {}", mergedAnomaly.getId(), e);
    }
    return anomalyDetails;
}

Example 39 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class AnomaliesResource method constructAnomaliesWrapperFromMergedAnomalies.

/**
   * Constructs AnomaliesWrapper object from a list of merged anomalies
   * @param mergedAnomalies
   * @return
   * @throws ExecutionException
   */
private AnomaliesWrapper constructAnomaliesWrapperFromMergedAnomalies(List<MergedAnomalyResultDTO> mergedAnomalies, int pageNumber) throws ExecutionException {
    AnomaliesWrapper anomaliesWrapper = new AnomaliesWrapper();
    anomaliesWrapper.setTotalAnomalies(mergedAnomalies.size());
    LOG.info("Total anomalies: {}", mergedAnomalies.size());
    // TODO: get page number and page size from client
    int pageSize = DEFAULT_PAGE_SIZE;
    int maxPageNumber = (mergedAnomalies.size() - 1) / pageSize + 1;
    if (pageNumber > maxPageNumber) {
        pageNumber = maxPageNumber;
    }
    if (pageNumber < 1) {
        pageNumber = 1;
    }
    int fromIndex = (pageNumber - 1) * pageSize;
    int toIndex = pageNumber * pageSize;
    if (toIndex > mergedAnomalies.size()) {
        toIndex = mergedAnomalies.size();
    }
    // Show most recent anomalies first, i.e., the anomaly whose end time is most recent then largest id shown at top
    Collections.sort(mergedAnomalies, new MergedAnomalyEndTimeComparator().reversed());
    List<MergedAnomalyResultDTO> displayedAnomalies = mergedAnomalies.subList(fromIndex, toIndex);
    anomaliesWrapper.setNumAnomaliesOnPage(displayedAnomalies.size());
    LOG.info("Page number: {} Page size: {} Num anomalies on page: {}", pageNumber, pageSize, displayedAnomalies.size());
    // for each anomaly, create anomaly details
    List<Future<AnomalyDetails>> anomalyDetailsListFutures = new ArrayList<>();
    for (MergedAnomalyResultDTO mergedAnomaly : displayedAnomalies) {
        Callable<AnomalyDetails> callable = new Callable<AnomalyDetails>() {

            @Override
            public AnomalyDetails call() throws Exception {
                String dataset = mergedAnomaly.getCollection();
                DatasetConfigDTO datasetConfig = CACHE_REGISTRY.getDatasetConfigCache().get(dataset);
                DateTimeFormatter timeSeriesDateFormatter = DateTimeFormat.forPattern(TIME_SERIES_DATE_FORMAT).withZone(Utils.getDataTimeZone(dataset));
                DateTimeFormatter startEndDateFormatterDays = DateTimeFormat.forPattern(START_END_DATE_FORMAT_DAYS).withZone(Utils.getDataTimeZone(dataset));
                DateTimeFormatter startEndDateFormatterHours = DateTimeFormat.forPattern(START_END_DATE_FORMAT_HOURS).withZone(Utils.getDataTimeZone(dataset));
                return getAnomalyDetails(mergedAnomaly, datasetConfig, timeSeriesDateFormatter, startEndDateFormatterHours, startEndDateFormatterDays, getExternalURL(mergedAnomaly));
            }
        };
        anomalyDetailsListFutures.add(threadPool.submit(callable));
    }
    List<AnomalyDetails> anomalyDetailsList = new ArrayList<>();
    for (Future<AnomalyDetails> anomalyDetailsFuture : anomalyDetailsListFutures) {
        try {
            AnomalyDetails anomalyDetails = anomalyDetailsFuture.get(120, TimeUnit.SECONDS);
            if (anomalyDetails != null) {
                anomalyDetailsList.add(anomalyDetails);
            }
        } catch (InterruptedException | ExecutionException | TimeoutException e) {
            LOG.error("Exception in getting AnomalyDetails", e);
        }
    }
    anomaliesWrapper.setAnomalyDetailsList(anomalyDetailsList);
    return anomaliesWrapper;
}

Example 40 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class MergedAnomalyResultManagerImpl method findById.

public MergedAnomalyResultDTO findById(Long id, boolean loadRawAnomalies) {
    MergedAnomalyResultBean mergedAnomalyResultBean = genericPojoDao.get(id, MergedAnomalyResultBean.class);
    if (mergedAnomalyResultBean != null) {
        MergedAnomalyResultDTO mergedAnomalyResultDTO;
        mergedAnomalyResultDTO = convertMergedAnomalyBean2DTO(mergedAnomalyResultBean, loadRawAnomalies);
        return mergedAnomalyResultDTO;
    } else {
        return null;
    }
}