Examples with PredictionModel com.linkedin.thirdeye.anomalydetection.model.prediction.PredictionModel used on opensource projects

Example 1 with PredictionModel

use of com.linkedin.thirdeye.anomalydetection.model.prediction.PredictionModel in project pinot by linkedin.

the class SimpleThresholdDetectionModel method detect.

@Override
public List<RawAnomalyResultDTO> detect(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    // Get thresholds
    double changeThreshold = Double.valueOf(getProperties().getProperty(CHANGE_THRESHOLD));
    double volumeThreshold = 0d;
    if (getProperties().containsKey(AVERAGE_VOLUME_THRESHOLD)) {
        volumeThreshold = Double.valueOf(getProperties().getProperty(AVERAGE_VOLUME_THRESHOLD));
    }
    long bucketSizeInMillis = anomalyDetectionContext.getBucketSizeInMS();
    // Compute the weight of this time series (average across whole)
    TimeSeries currentTimeSeries = anomalyDetectionContext.getTransformedCurrent(metricName);
    double averageValue = 0;
    for (long time : currentTimeSeries.timestampSet()) {
        averageValue += currentTimeSeries.get(time);
    }
    Interval currentInterval = currentTimeSeries.getTimeSeriesInterval();
    long currentStart = currentInterval.getStartMillis();
    long currentEnd = currentInterval.getEndMillis();
    long numBuckets = (currentEnd - currentStart) / bucketSizeInMillis;
    if (numBuckets != 0) {
        averageValue /= numBuckets;
    }
    // Check if this time series even meets our volume threshold
    DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
    if (averageValue < volumeThreshold) {
        LOGGER.info("{} does not meet volume threshold {}: {}", dimensionMap, volumeThreshold, averageValue);
        // empty list
        return anomalyResults;
    }
    PredictionModel predictionModel = anomalyDetectionContext.getTrainedPredictionModel(metricName);
    if (!(predictionModel instanceof ExpectedTimeSeriesPredictionModel)) {
        LOGGER.info("SimpleThresholdDetectionModel detection model expects an ExpectedTimeSeriesPredictionModel but the trained prediction model in anomaly detection context is not.");
        // empty list
        return anomalyResults;
    }
    ExpectedTimeSeriesPredictionModel expectedTimeSeriesPredictionModel = (ExpectedTimeSeriesPredictionModel) predictionModel;
    TimeSeries expectedTimeSeries = expectedTimeSeriesPredictionModel.getExpectedTimeSeries();
    Interval expectedTSInterval = expectedTimeSeries.getTimeSeriesInterval();
    long expectedStart = expectedTSInterval.getStartMillis();
    long seasonalOffset = currentStart - expectedStart;
    for (long currentTimestamp : currentTimeSeries.timestampSet()) {
        long expectedTimestamp = currentTimestamp - seasonalOffset;
        if (!expectedTimeSeries.hasTimestamp(expectedTimestamp)) {
            continue;
        }
        double baselineValue = expectedTimeSeries.get(expectedTimestamp);
        double currentValue = currentTimeSeries.get(currentTimestamp);
        if (isAnomaly(currentValue, baselineValue, changeThreshold)) {
            RawAnomalyResultDTO anomalyResult = new RawAnomalyResultDTO();
            anomalyResult.setDimensions(dimensionMap);
            anomalyResult.setProperties(getProperties().toString());
            anomalyResult.setStartTime(currentTimestamp);
            // point-in-time
            anomalyResult.setEndTime(currentTimestamp + bucketSizeInMillis);
            anomalyResult.setScore(averageValue);
            anomalyResult.setWeight(calculateChange(currentValue, baselineValue));
            anomalyResult.setAvgCurrentVal(currentValue);
            anomalyResult.setAvgBaselineVal(baselineValue);
            String message = getAnomalyResultMessage(changeThreshold, currentValue, baselineValue);
            anomalyResult.setMessage(message);
            anomalyResults.add(anomalyResult);
            if (currentValue == 0.0 || baselineValue == 0.0) {
                anomalyResult.setDataMissing(true);
            }
        }
    }
    return anomalyResults;
}

Example 2 with PredictionModel

use of com.linkedin.thirdeye.anomalydetection.model.prediction.PredictionModel in project pinot by linkedin.

the class AbstractModularizedAnomalyFunction method transformAndPredictTimeSeries.

/**
   * Performs the following operations on the given an anomaly detection context.
   * 1. Transform current and baseline time series.
   * 2. Train prediction model using the baseline time series.
   *
   * At the end of this method, a transformed current time series, transformed baselines, and a
   * trained prediction model are appended to the given anomaly detection context.
   *
   * The processed anomaly detection context has multiple usages. For example, it could be used for
   * detecting anomalies, plotting UI, updating the information of anomalies, etc.
   *
   * @param metricName the name of the metric on which we apply transformation and prediction
   * @param anomalyDetectionContext anomaly detection context that contains the necessary time
   *                                series for preparing the prediction model
   */
public void transformAndPredictTimeSeries(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    transformTimeSeries(metricName, anomalyDetectionContext);
    // Train Prediction Model
    PredictionModel predictionModel = getPredictionModel();
    predictionModel.train(anomalyDetectionContext.getTransformedBaselines(metricName), anomalyDetectionContext);
    anomalyDetectionContext.setTrainedPredictionModel(metricName, predictionModel);
}

Example 3 with PredictionModel

use of com.linkedin.thirdeye.anomalydetection.model.prediction.PredictionModel in project pinot by linkedin.

the class SimplePercentageMergeModel method update.

/**
   * The weight of the merged anomaly is calculated by this equation:
   *     weight = (avg. observed value) / (avg. expected value) - 1;
   *
   * Note that the values of the holes in the time series are not included in the computation.
   * Considering the observed and expected time series:
   *    observed:  1 2 x 4 x 6
   *    expected:  1 x x 4 5 6
   * The values that are included in the computation are those at slots 1, 4, and 6.
   *
   * @param anomalyDetectionContext the context that provided a trained
   *                                ExpectedTimeSeriesPredictionModel for computing the weight.
   *                                Moreover, the data range of the time series should equals the
   *                                range of anomaly to be updated.
   *
   * @param anomalyToUpdated the anomaly of which the information is updated.
   */
@Override
public void update(AnomalyDetectionContext anomalyDetectionContext, MergedAnomalyResultDTO anomalyToUpdated) {
    String mainMetric = anomalyDetectionContext.getAnomalyDetectionFunction().getSpec().getTopicMetric();
    PredictionModel predictionModel = anomalyDetectionContext.getTrainedPredictionModel(mainMetric);
    if (!(predictionModel instanceof ExpectedTimeSeriesPredictionModel)) {
        LOGGER.error("SimplePercentageMergeModel expects an ExpectedTimeSeriesPredictionModel but the trained model is not one.");
        return;
    }
    ExpectedTimeSeriesPredictionModel expectedTimeSeriesPredictionModel = (ExpectedTimeSeriesPredictionModel) predictionModel;
    TimeSeries expectedTimeSeries = expectedTimeSeriesPredictionModel.getExpectedTimeSeries();
    long expectedStartTime = expectedTimeSeries.getTimeSeriesInterval().getStartMillis();
    TimeSeries observedTimeSeries = anomalyDetectionContext.getTransformedCurrent(mainMetric);
    long observedStartTime = observedTimeSeries.getTimeSeriesInterval().getStartMillis();
    double avgCurrent = 0d;
    double avgBaseline = 0d;
    int count = 0;
    Interval anomalyInterval = new Interval(anomalyToUpdated.getStartTime(), anomalyToUpdated.getEndTime());
    for (long observedTimestamp : observedTimeSeries.timestampSet()) {
        if (anomalyInterval.contains(observedTimestamp)) {
            long offset = observedTimestamp - observedStartTime;
            long expectedTimestamp = expectedStartTime + offset;
            if (expectedTimeSeries.hasTimestamp(expectedTimestamp)) {
                avgCurrent += observedTimeSeries.get(observedTimestamp);
                avgBaseline += expectedTimeSeries.get(expectedTimestamp);
                ++count;
            }
        }
    }
    double weight = 0d;
    if (count != 0 && avgBaseline != 0d) {
        weight = (avgCurrent - avgBaseline) / avgBaseline;
        avgCurrent /= count;
        avgBaseline /= count;
    } else {
        weight = 0d;
    }
    // Average score of raw anomalies
    List<RawAnomalyResultDTO> rawAnomalyResultDTOs = anomalyToUpdated.getAnomalyResults();
    double score = 0d;
    if (CollectionUtils.isNotEmpty(rawAnomalyResultDTOs)) {
        for (RawAnomalyResultDTO rawAnomaly : rawAnomalyResultDTOs) {
            score += rawAnomaly.getScore();
        }
        score /= rawAnomalyResultDTOs.size();
    } else {
        score = anomalyToUpdated.getScore();
    }
    anomalyToUpdated.setWeight(weight);
    anomalyToUpdated.setScore(score);
    anomalyToUpdated.setAvgCurrentVal(avgCurrent);
    anomalyToUpdated.setAvgBaselineVal(avgBaseline);
    anomalyToUpdated.setMessage(String.format(DEFAULT_MESSAGE_TEMPLATE, weight * 100, avgCurrent, avgBaseline, score));
}