Examples with MergedAnomalyResultDTO com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO used on opensource projects

Example 1 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class HistoricalAnomalyEventProvider method getEvents.

@Override
public List<EventDTO> getEvents(EventFilter eventFilter) {
    List<MergedAnomalyResultDTO> mergedAnomalies;
    List<EventDTO> events = new ArrayList<>();
    if (eventFilter.getMetricName() == null) {
        mergedAnomalies = mergedAnomalyDAO.findByTime(eventFilter.getStartTime(), eventFilter.getStartTime());
    } else {
        mergedAnomalies = mergedAnomalyDAO.findByMetricTime(eventFilter.getMetricName(), eventFilter.getStartTime(), eventFilter.getStartTime(), false);
    }
    if (mergedAnomalies != null) {
        for (MergedAnomalyResultDTO mergedAnomalyResultDTO : mergedAnomalies) {
            events.add(getAnomalyEvent(mergedAnomalyResultDTO));
        }
    }
    return events;
}

Example 2 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class AnomalyMergeExecutor method updateMergedAnomalyWeight.

/**
   * Uses function-specific method to re-computes the weight of merged anomaly.
   *
   * @param anomalyMergedResult the merged anomaly to be updated
   * @param mergeConfig the merge configuration that was applied when merge the merged anomaly
   * @throws Exception if error occurs when retrieving the time series for calculating the weight
   */
private void updateMergedAnomalyWeight(MergedAnomalyResultDTO anomalyMergedResult, AnomalyMergeConfig mergeConfig) throws Exception {
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyMergedResult.getFunction();
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(anomalyMergedResult.getStartTime(), anomalyMergedResult.getEndTime());
    TimeGranularity timeGranularity = new TimeGranularity(anomalyFunctionSpec.getBucketSize(), anomalyFunctionSpec.getBucketUnit());
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(anomalyFunctionSpec, startEndTimeRanges, anomalyMergedResult.getDimensions(), timeGranularity, false);
    if (metricTimeSeries != null) {
        DateTime windowStart = new DateTime(anomalyMergedResult.getStartTime());
        DateTime windowEnd = new DateTime(anomalyMergedResult.getEndTime());
        List<MergedAnomalyResultDTO> knownAnomalies = Collections.emptyList();
        // Retrieve history merged anomalies
        if (anomalyFunction.useHistoryAnomaly()) {
            switch(mergeConfig.getMergeStrategy()) {
                case FUNCTION:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis());
                    break;
                case FUNCTION_DIMENSIONS:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis(), anomalyMergedResult.getDimensions());
                    break;
                default:
                    throw new IllegalArgumentException("Merge strategy " + mergeConfig.getMergeStrategy() + " not supported");
            }
            if (knownAnomalies.size() > 0) {
                LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
                LOG.info("Checking if any known anomalies overlap with the monitoring window of anomaly detection, which could result in unwanted holes in current values.");
                AnomalyUtils.logAnomaliesOverlapWithWindow(windowStart, windowEnd, knownAnomalies);
            }
        }
        // Transform Time Series
        List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getTopicMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, windowStart.getMillis(), scalingFactors, anomalyFunctionSpec.getTopicMetric(), properties);
        }
        anomalyFunction.updateMergedAnomalyInfo(anomalyMergedResult, metricTimeSeries, windowStart, windowEnd, knownAnomalies);
    }
}

Example 3 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class AnomalyMergeExecutor method performMergeBasedOnFunctionId.

@Deprecated
private void performMergeBasedOnFunctionId(AnomalyFunctionDTO function, AnomalyMergeConfig mergeConfig, List<RawAnomalyResultDTO> unmergedResults, List<MergedAnomalyResultDTO> output) {
    // Now find last MergedAnomalyResult in same category
    MergedAnomalyResultDTO latestMergedResult = mergedResultDAO.findLatestByFunctionIdOnly(function.getId());
    // TODO : get mergeConfig from function
    List<MergedAnomalyResultDTO> mergedResults = AnomalyTimeBasedSummarizer.mergeAnomalies(latestMergedResult, unmergedResults, mergeConfig.getMaxMergeDurationLength(), mergeConfig.getSequentialAllowedGap());
    for (MergedAnomalyResultDTO mergedResult : mergedResults) {
        mergedResult.setFunction(function);
    }
    LOG.info("Merging [{}] raw anomalies into [{}] merged anomalies for function id : [{}]", unmergedResults.size(), mergedResults.size(), function.getId());
    output.addAll(mergedResults);
}

Example 4 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class AnomalyTimeBasedSummarizer method mergeAnomalies.

/**
   * @param mergedAnomaly : last merged anomaly
   * @param anomalies     : list of raw anomalies to be merged with last mergedAnomaly
   * @param maxMergedDurationMillis   : length of a merged anomaly
   * @param sequentialAllowedGap : allowed gap between two raw anomalies in order to merge
   * @return
   */
public static List<MergedAnomalyResultDTO> mergeAnomalies(MergedAnomalyResultDTO mergedAnomaly, List<RawAnomalyResultDTO> anomalies, long maxMergedDurationMillis, long sequentialAllowedGap) {
    // sort anomalies in natural order of start time
    Collections.sort(anomalies, (o1, o2) -> (int) ((o1.getStartTime() - o2.getStartTime()) / 1000));
    boolean applySequentialGapBasedSplit = false;
    boolean applyMaxDurationBasedSplit = false;
    if (maxMergedDurationMillis > 0) {
        applyMaxDurationBasedSplit = true;
    }
    if (sequentialAllowedGap > 0) {
        applySequentialGapBasedSplit = true;
    }
    List<MergedAnomalyResultDTO> mergedAnomalies = new ArrayList<>();
    for (int i = 0; i < anomalies.size(); i++) {
        RawAnomalyResultDTO currentResult = anomalies.get(i);
        if (mergedAnomaly == null || currentResult.getEndTime() < mergedAnomaly.getStartTime()) {
            mergedAnomaly = new MergedAnomalyResultDTO();
            populateMergedResult(mergedAnomaly, currentResult);
        } else {
            // compare current with merged and decide whether to merge the current result or create a new one
            if (applySequentialGapBasedSplit && (currentResult.getStartTime() - mergedAnomaly.getEndTime()) > sequentialAllowedGap) {
                // Split here
                // add previous merged result
                mergedAnomalies.add(mergedAnomaly);
                //set current raw result
                mergedAnomaly = new MergedAnomalyResultDTO();
                populateMergedResult(mergedAnomaly, currentResult);
            } else {
                // add the current raw result into mergedResult
                if (currentResult.getStartTime() < mergedAnomaly.getStartTime()) {
                    mergedAnomaly.setStartTime(currentResult.getStartTime());
                }
                if (currentResult.getEndTime() > mergedAnomaly.getEndTime()) {
                    mergedAnomaly.setEndTime(currentResult.getEndTime());
                }
                if (!mergedAnomaly.getAnomalyResults().contains(currentResult)) {
                    mergedAnomaly.getAnomalyResults().add(currentResult);
                    currentResult.setMerged(true);
                }
            }
        }
        // till this point merged result contains current raw result
        if (applyMaxDurationBasedSplit && // check if Max Duration for merged has passed, if so, create new one
        mergedAnomaly.getEndTime() - mergedAnomaly.getStartTime() >= maxMergedDurationMillis) {
            // check if next anomaly has same start time as current one, that should be merged with current one too
            if (i < (anomalies.size() - 1) && anomalies.get(i + 1).getStartTime().equals(currentResult.getStartTime())) {
            // no need to split as we want to include the next raw anomaly into the current one
            } else {
                // Split here
                mergedAnomalies.add(mergedAnomaly);
                mergedAnomaly = null;
            }
        }
        if (i == (anomalies.size() - 1) && mergedAnomaly != null) {
            mergedAnomalies.add(mergedAnomaly);
        }
    }
    LOG.info("merging [{}] raw anomalies", anomalies.size());
    return mergedAnomalies;
}

Example 5 with MergedAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO in project pinot by linkedin.

the class TimeBasedAnomalyMerger method fetchDataByDimension.

/**
   * Fetch time series, known merged anomalies, and scaling factor for the specified dimension. Note that scaling
   * factor has no dimension information, so all scaling factor in the specified time range will be retrieved.
   *
   * @param windowStartTime the start time for retrieving the data
   * @param windowEndTime the end time for retrieving the data
   * @param dimensions the dimension of the data
   * @param anomalyFunction the anomaly function that produces the anomaly
   * @param mergedResultDAO DAO for merged anomalies
   * @param overrideConfigDAO DAO for override configuration
   * @param endTimeInclusive set to true if the end time should be inclusive; mainly used by the queries from UI
   * @return an anomaly detection input context that contains all the retrieved data
   * @throws Exception if it fails to retrieve time series from DB.
   */
public static AnomalyDetectionInputContext fetchDataByDimension(long windowStartTime, long windowEndTime, DimensionMap dimensions, BaseAnomalyFunction anomalyFunction, MergedAnomalyResultManager mergedResultDAO, OverrideConfigManager overrideConfigDAO, boolean endTimeInclusive) throws Exception {
    AnomalyFunctionDTO functionSpec = anomalyFunction.getSpec();
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime);
    TimeGranularity timeGranularity = new TimeGranularity(functionSpec.getBucketSize(), functionSpec.getBucketUnit());
    AnomalyDetectionInputContext adInputContext = new AnomalyDetectionInputContext();
    // Retrieve Time Series
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(functionSpec, startEndTimeRanges, dimensions, timeGranularity, endTimeInclusive);
    Map<DimensionMap, MetricTimeSeries> metricTimeSeriesMap = new HashMap<>();
    metricTimeSeriesMap.put(dimensions, metricTimeSeries);
    adInputContext.setDimensionKeyMetricTimeSeriesMap(metricTimeSeriesMap);
    // Retrieve historical anomaly
    if (anomalyFunction.useHistoryAnomaly()) {
        List<MergedAnomalyResultDTO> knownAnomalies = getBaselineKnownAnomaliesByDimension(anomalyFunction, windowStartTime, windowEndTime, dimensions, mergedResultDAO);
        ListMultimap<DimensionMap, MergedAnomalyResultDTO> mergedAnomalyMap = ArrayListMultimap.create();
        mergedAnomalyMap.putAll(dimensions, knownAnomalies);
        adInputContext.setKnownMergedAnomalies(mergedAnomalyMap);
        if (knownAnomalies.size() > 0) {
            LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
        }
    }
    // Retrieve scaling factor
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, functionSpec.getCollection(), functionSpec.getTopicMetric(), functionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime));
    adInputContext.setScalingFactors(scalingFactors);
    return adInputContext;
}