Examples with RawAnomalyResultDTO com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO used on opensource projects

Example 6 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class DetectionTaskRunner method storeData.

private void storeData(AnomalyDetectionOutputContext anomalyDetectionOutputContext) {
    RawAnomalyResultManager rawAnomalyDAO = DAO_REGISTRY.getRawAnomalyResultDAO();
    MergedAnomalyResultManager mergedAmomalyDAO = DAO_REGISTRY.getMergedAnomalyResultDAO();
    for (RawAnomalyResultDTO rawAnomalyResultDTO : anomalyDetectionOutputContext.getRawAnomalies().values()) {
        rawAnomalyDAO.save(rawAnomalyResultDTO);
    }
    for (MergedAnomalyResultDTO mergedAnomalyResultDTO : anomalyDetectionOutputContext.getMergedAnomalies().values()) {
        mergedAmomalyDAO.update(mergedAnomalyResultDTO);
    }
}

Example 7 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class DetectionTaskRunner method runAnalyze.

private List<RawAnomalyResultDTO> runAnalyze(DateTime windowStart, DateTime windowEnd, AnomalyDetectionInputContext anomalyDetectionInputContext, DimensionMap dimensionMap) {
    String metricName = anomalyFunction.getSpec().getTopicMetric();
    MetricTimeSeries metricTimeSeries = anomalyDetectionInputContext.getDimensionKeyMetricTimeSeriesMap().get(dimensionMap);
    // Get current entry's knownMergedAnomalies, which should have the same explored dimensions
    List<MergedAnomalyResultDTO> knownMergedAnomaliesOfAnEntry = anomalyDetectionInputContext.getKnownMergedAnomalies().get(dimensionMap);
    List<MergedAnomalyResultDTO> historyMergedAnomalies;
    if (anomalyFunction.useHistoryAnomaly()) {
        historyMergedAnomalies = retainHistoryMergedAnomalies(windowStart.getMillis(), knownMergedAnomaliesOfAnEntry);
    } else {
        historyMergedAnomalies = Collections.emptyList();
    }
    LOG.info("Analyzing anomaly function with explored dimensions: {}, windowStart: {}, windowEnd: {}", dimensionMap, windowStart, windowEnd);
    AnomalyUtils.logAnomaliesOverlapWithWindow(windowStart, windowEnd, historyMergedAnomalies);
    List<RawAnomalyResultDTO> resultsOfAnEntry = Collections.emptyList();
    try {
        // Run algorithm
        // Scaling time series according to the scaling factor
        List<ScalingFactor> scalingFactors = anomalyDetectionInputContext.getScalingFactors();
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, windowStart.getMillis(), scalingFactors, metricName, properties);
        }
        resultsOfAnEntry = anomalyFunction.analyze(dimensionMap, metricTimeSeries, windowStart, windowEnd, historyMergedAnomalies);
    } catch (Exception e) {
        LOG.error("Could not compute for {}", dimensionMap, e);
    }
    // Remove detected anomalies that have existed in database
    if (CollectionUtils.isNotEmpty(resultsOfAnEntry)) {
        List<RawAnomalyResultDTO> existingRawAnomaliesOfAnEntry = anomalyDetectionInputContext.getExistingRawAnomalies().get(dimensionMap);
        resultsOfAnEntry = removeFromExistingRawAnomalies(resultsOfAnEntry, existingRawAnomaliesOfAnEntry);
    }
    if (CollectionUtils.isNotEmpty(resultsOfAnEntry)) {
        List<MergedAnomalyResultDTO> existingMergedAnomalies = retainExistingMergedAnomalies(windowStart.getMillis(), windowEnd.getMillis(), knownMergedAnomaliesOfAnEntry);
        resultsOfAnEntry = removeFromExistingMergedAnomalies(resultsOfAnEntry, existingMergedAnomalies);
    }
    return resultsOfAnEntry;
}

Example 8 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class MinMaxThresholdDetectionModel method detect.

@Override
public List<RawAnomalyResultDTO> detect(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    // Get min / max props
    Double min = null;
    if (properties.containsKey(MIN_VAL)) {
        min = Double.valueOf(properties.getProperty(MIN_VAL));
    }
    Double max = null;
    if (properties.containsKey(MAX_VAL)) {
        max = Double.valueOf(properties.getProperty(MAX_VAL));
    }
    TimeSeries timeSeries = anomalyDetectionContext.getTransformedCurrent(metricName);
    // Compute the weight of this time series (average across whole)
    double averageValue = 0;
    for (long time : timeSeries.timestampSet()) {
        averageValue += timeSeries.get(time);
    }
    // Compute the bucket size, so we can iterate in those steps
    long bucketMillis = anomalyDetectionContext.getBucketSizeInMS();
    Interval timeSeriesInterval = timeSeries.getTimeSeriesInterval();
    long numBuckets = Math.abs(timeSeriesInterval.getEndMillis() - timeSeriesInterval.getStartMillis()) / bucketMillis;
    // avg value of this time series
    averageValue /= numBuckets;
    DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
    for (long timeBucket : timeSeries.timestampSet()) {
        double value = timeSeries.get(timeBucket);
        double deviationFromThreshold = getDeviationFromThreshold(value, min, max);
        if (deviationFromThreshold != 0) {
            RawAnomalyResultDTO anomalyResult = new RawAnomalyResultDTO();
            anomalyResult.setProperties(properties.toString());
            anomalyResult.setStartTime(timeBucket);
            // point-in-time
            anomalyResult.setEndTime(timeBucket + bucketMillis);
            anomalyResult.setDimensions(dimensionMap);
            anomalyResult.setScore(averageValue);
            // higher change, higher the severity
            anomalyResult.setWeight(deviationFromThreshold);
            anomalyResult.setAvgCurrentVal(value);
            String message = String.format(DEFAULT_MESSAGE_TEMPLATE, deviationFromThreshold, value, min, max);
            anomalyResult.setMessage(message);
            if (value == 0.0) {
                anomalyResult.setDataMissing(true);
            }
            anomalyResults.add(anomalyResult);
        }
    }
    return anomalyResults;
}

Example 9 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class SimpleThresholdDetectionModel method detect.

@Override
public List<RawAnomalyResultDTO> detect(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    // Get thresholds
    double changeThreshold = Double.valueOf(getProperties().getProperty(CHANGE_THRESHOLD));
    double volumeThreshold = 0d;
    if (getProperties().containsKey(AVERAGE_VOLUME_THRESHOLD)) {
        volumeThreshold = Double.valueOf(getProperties().getProperty(AVERAGE_VOLUME_THRESHOLD));
    }
    long bucketSizeInMillis = anomalyDetectionContext.getBucketSizeInMS();
    // Compute the weight of this time series (average across whole)
    TimeSeries currentTimeSeries = anomalyDetectionContext.getTransformedCurrent(metricName);
    double averageValue = 0;
    for (long time : currentTimeSeries.timestampSet()) {
        averageValue += currentTimeSeries.get(time);
    }
    Interval currentInterval = currentTimeSeries.getTimeSeriesInterval();
    long currentStart = currentInterval.getStartMillis();
    long currentEnd = currentInterval.getEndMillis();
    long numBuckets = (currentEnd - currentStart) / bucketSizeInMillis;
    if (numBuckets != 0) {
        averageValue /= numBuckets;
    }
    // Check if this time series even meets our volume threshold
    DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
    if (averageValue < volumeThreshold) {
        LOGGER.info("{} does not meet volume threshold {}: {}", dimensionMap, volumeThreshold, averageValue);
        // empty list
        return anomalyResults;
    }
    PredictionModel predictionModel = anomalyDetectionContext.getTrainedPredictionModel(metricName);
    if (!(predictionModel instanceof ExpectedTimeSeriesPredictionModel)) {
        LOGGER.info("SimpleThresholdDetectionModel detection model expects an ExpectedTimeSeriesPredictionModel but the trained prediction model in anomaly detection context is not.");
        // empty list
        return anomalyResults;
    }
    ExpectedTimeSeriesPredictionModel expectedTimeSeriesPredictionModel = (ExpectedTimeSeriesPredictionModel) predictionModel;
    TimeSeries expectedTimeSeries = expectedTimeSeriesPredictionModel.getExpectedTimeSeries();
    Interval expectedTSInterval = expectedTimeSeries.getTimeSeriesInterval();
    long expectedStart = expectedTSInterval.getStartMillis();
    long seasonalOffset = currentStart - expectedStart;
    for (long currentTimestamp : currentTimeSeries.timestampSet()) {
        long expectedTimestamp = currentTimestamp - seasonalOffset;
        if (!expectedTimeSeries.hasTimestamp(expectedTimestamp)) {
            continue;
        }
        double baselineValue = expectedTimeSeries.get(expectedTimestamp);
        double currentValue = currentTimeSeries.get(currentTimestamp);
        if (isAnomaly(currentValue, baselineValue, changeThreshold)) {
            RawAnomalyResultDTO anomalyResult = new RawAnomalyResultDTO();
            anomalyResult.setDimensions(dimensionMap);
            anomalyResult.setProperties(getProperties().toString());
            anomalyResult.setStartTime(currentTimestamp);
            // point-in-time
            anomalyResult.setEndTime(currentTimestamp + bucketSizeInMillis);
            anomalyResult.setScore(averageValue);
            anomalyResult.setWeight(calculateChange(currentValue, baselineValue));
            anomalyResult.setAvgCurrentVal(currentValue);
            anomalyResult.setAvgBaselineVal(baselineValue);
            String message = getAnomalyResultMessage(changeThreshold, currentValue, baselineValue);
            anomalyResult.setMessage(message);
            anomalyResults.add(anomalyResult);
            if (currentValue == 0.0 || baselineValue == 0.0) {
                anomalyResult.setDataMissing(true);
            }
        }
    }
    return anomalyResults;
}

Example 10 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class AverageAnomalyMergeModel method update.

/**
   * The weight and score is the average weight and score, respectively, of the raw anomalies of
   * the given merged anomaly. If the merged anomaly could not provides the list of its raw
   * anomalies, then weight and score are set to 0d.
   *
   * @param anomalyDetectionContext a context that would not be used.
   *
   * @param anomalyToUpdated the anomaly of which the information is updated.
   */
@Override
public void update(AnomalyDetectionContext anomalyDetectionContext, MergedAnomalyResultDTO anomalyToUpdated) {
    if (CollectionUtils.isEmpty(anomalyToUpdated.getAnomalyResults())) {
        return;
    }
    List<RawAnomalyResultDTO> rawAnomalyResultDTOs = anomalyToUpdated.getAnomalyResults();
    double weight = 0d;
    double score = 0d;
    double avgBaseline = 0d;
    double avgCurrent = 0d;
    for (RawAnomalyResultDTO rawAnomaly : rawAnomalyResultDTOs) {
        weight += rawAnomaly.getWeight();
        score += rawAnomaly.getScore();
        avgCurrent += rawAnomaly.getAvgCurrentVal();
        avgBaseline += rawAnomaly.getAvgBaselineVal();
    }
    if (rawAnomalyResultDTOs.size() != 0) {
        double size = rawAnomalyResultDTOs.size();
        weight /= size;
        score /= size;
        avgCurrent /= size;
        anomalyToUpdated.setAvgCurrentVal(avgCurrent);
        avgBaseline /= size;
        anomalyToUpdated.setAvgBaselineVal(avgBaseline);
    }
    anomalyToUpdated.setWeight(weight);
    anomalyToUpdated.setScore(score);
    anomalyToUpdated.setAvgCurrentVal(avgCurrent);
    anomalyToUpdated.setAvgBaselineVal(avgBaseline);
    anomalyToUpdated.setMessage(String.format(DEFAULT_MESSAGE_TEMPLATE, avgBaseline, avgCurrent, weight, score));
}