Examples with TimeSeries com.linkedin.thirdeye.anomalydetection.context.TimeSeries used on opensource projects

Example 6 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class AbstractModularizedAnomalyFunction method getTimeSeriesView.

// TODO: Generate time series view using ViewModel
@Override
public AnomalyTimelinesView getTimeSeriesView(MetricTimeSeries timeSeries, long bucketMillis, String metric, long viewWindowStartTime, long viewWindowEndTime, List<MergedAnomalyResultDTO> knownAnomalies) {
    AnomalyDetectionContext anomalyDetectionContext = BackwardAnomalyFunctionUtils.buildAnomalyDetectionContext(this, timeSeries, spec.getTopicMetric(), null, spec.getBucketSize(), spec.getBucketUnit(), new DateTime(viewWindowStartTime), new DateTime(viewWindowEndTime));
    String mainMetric = anomalyDetectionContext.getAnomalyDetectionFunction().getSpec().getTopicMetric();
    this.transformAndPredictTimeSeries(mainMetric, anomalyDetectionContext);
    TimeSeries observedTS = anomalyDetectionContext.getTransformedCurrent(mainMetric);
    TimeSeries expectedTS = ((ExpectedTimeSeriesPredictionModel) anomalyDetectionContext.getTrainedPredictionModel(mainMetric)).getExpectedTimeSeries();
    long expectedTSStartTime = expectedTS.getTimeSeriesInterval().getStartMillis();
    // Construct AnomalyTimelinesView
    AnomalyTimelinesView anomalyTimelinesView = new AnomalyTimelinesView();
    int bucketCount = (int) ((viewWindowEndTime - viewWindowStartTime) / bucketMillis);
    for (int i = 0; i < bucketCount; ++i) {
        long currentBucketMillis = viewWindowStartTime + i * bucketMillis;
        long baselineBucketMillis = expectedTSStartTime + i * bucketMillis;
        double observedValue = 0d;
        if (observedTS.hasTimestamp(currentBucketMillis)) {
            observedValue = observedTS.get(currentBucketMillis);
        }
        double expectedValue = 0d;
        if (expectedTS.hasTimestamp(baselineBucketMillis)) {
            expectedValue = expectedTS.get(baselineBucketMillis);
        }
        TimeBucket timebucket = new TimeBucket(currentBucketMillis, currentBucketMillis + bucketMillis, baselineBucketMillis, baselineBucketMillis + bucketMillis);
        anomalyTimelinesView.addTimeBuckets(timebucket);
        anomalyTimelinesView.addCurrentValues(observedValue);
        anomalyTimelinesView.addBaselineValues(expectedValue);
    }
    return anomalyTimelinesView;
}

Example 7 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class SimplePercentageMergeModel method update.

/**
   * The weight of the merged anomaly is calculated by this equation:
   *     weight = (avg. observed value) / (avg. expected value) - 1;
   *
   * Note that the values of the holes in the time series are not included in the computation.
   * Considering the observed and expected time series:
   *    observed:  1 2 x 4 x 6
   *    expected:  1 x x 4 5 6
   * The values that are included in the computation are those at slots 1, 4, and 6.
   *
   * @param anomalyDetectionContext the context that provided a trained
   *                                ExpectedTimeSeriesPredictionModel for computing the weight.
   *                                Moreover, the data range of the time series should equals the
   *                                range of anomaly to be updated.
   *
   * @param anomalyToUpdated the anomaly of which the information is updated.
   */
@Override
public void update(AnomalyDetectionContext anomalyDetectionContext, MergedAnomalyResultDTO anomalyToUpdated) {
    String mainMetric = anomalyDetectionContext.getAnomalyDetectionFunction().getSpec().getTopicMetric();
    PredictionModel predictionModel = anomalyDetectionContext.getTrainedPredictionModel(mainMetric);
    if (!(predictionModel instanceof ExpectedTimeSeriesPredictionModel)) {
        LOGGER.error("SimplePercentageMergeModel expects an ExpectedTimeSeriesPredictionModel but the trained model is not one.");
        return;
    }
    ExpectedTimeSeriesPredictionModel expectedTimeSeriesPredictionModel = (ExpectedTimeSeriesPredictionModel) predictionModel;
    TimeSeries expectedTimeSeries = expectedTimeSeriesPredictionModel.getExpectedTimeSeries();
    long expectedStartTime = expectedTimeSeries.getTimeSeriesInterval().getStartMillis();
    TimeSeries observedTimeSeries = anomalyDetectionContext.getTransformedCurrent(mainMetric);
    long observedStartTime = observedTimeSeries.getTimeSeriesInterval().getStartMillis();
    double avgCurrent = 0d;
    double avgBaseline = 0d;
    int count = 0;
    Interval anomalyInterval = new Interval(anomalyToUpdated.getStartTime(), anomalyToUpdated.getEndTime());
    for (long observedTimestamp : observedTimeSeries.timestampSet()) {
        if (anomalyInterval.contains(observedTimestamp)) {
            long offset = observedTimestamp - observedStartTime;
            long expectedTimestamp = expectedStartTime + offset;
            if (expectedTimeSeries.hasTimestamp(expectedTimestamp)) {
                avgCurrent += observedTimeSeries.get(observedTimestamp);
                avgBaseline += expectedTimeSeries.get(expectedTimestamp);
                ++count;
            }
        }
    }
    double weight = 0d;
    if (count != 0 && avgBaseline != 0d) {
        weight = (avgCurrent - avgBaseline) / avgBaseline;
        avgCurrent /= count;
        avgBaseline /= count;
    } else {
        weight = 0d;
    }
    // Average score of raw anomalies
    List<RawAnomalyResultDTO> rawAnomalyResultDTOs = anomalyToUpdated.getAnomalyResults();
    double score = 0d;
    if (CollectionUtils.isNotEmpty(rawAnomalyResultDTOs)) {
        for (RawAnomalyResultDTO rawAnomaly : rawAnomalyResultDTOs) {
            score += rawAnomaly.getScore();
        }
        score /= rawAnomalyResultDTOs.size();
    } else {
        score = anomalyToUpdated.getScore();
    }
    anomalyToUpdated.setWeight(weight);
    anomalyToUpdated.setScore(score);
    anomalyToUpdated.setAvgCurrentVal(avgCurrent);
    anomalyToUpdated.setAvgBaselineVal(avgBaseline);
    anomalyToUpdated.setMessage(String.format(DEFAULT_MESSAGE_TEMPLATE, weight * 100, avgCurrent, avgBaseline, score));
}

Example 8 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class SeasonalAveragePredictionModel method getLatestTimeSeries.

/**
   * Returns the time series, which has the largest start millis, from a set of time series.
   *
   * @param baselineTimeSeries the set of baselines
   * @return the time series, which has the largest start millis, from a set of time series.
   */
private TimeSeries getLatestTimeSeries(List<TimeSeries> baselineTimeSeries) {
    if (CollectionUtils.isNotEmpty(baselineTimeSeries)) {
        if (baselineTimeSeries.size() > 1) {
            TimeSeries latestTimeSeries = baselineTimeSeries.get(0);
            Interval latestInterval = latestTimeSeries.getTimeSeriesInterval();
            for (TimeSeries ts : baselineTimeSeries) {
                Interval currentInterval = ts.getTimeSeriesInterval();
                if (latestInterval.getStartMillis() < currentInterval.getStartMillis()) {
                    latestTimeSeries = ts;
                    latestInterval = currentInterval;
                }
            }
            return latestTimeSeries;
        } else {
            return baselineTimeSeries.get(0);
        }
    } else {
        return null;
    }
}

Example 9 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class MovingAverageSmoothingFunction method transform.

/**
   * Smooths the given time series using moving average.
   *
   * If the input time series is shorter than the moving average window size, then this method
   * does not apply smoothing on the time series, i.e., it returns the original time series.
   *
   * The transformed time series is shorten by the size of the moving average window in
   * comparison to the original time series. For instance, if there are 10 consecutive data points
   * the a time series and the window size for moving average is 2, then the transformed time series
   * contains only 9 consecutive data points; The first data points has no other data point to
   * average and thus it is discarded.
   *
   * @param timeSeries the time series that provides the data points to be transformed.
   * @param anomalyDetectionContext the anomaly detection context that could provide additional
   *                                information for the transformation.
   * @return a time series that is smoothed using moving average.
   */
@Override
public TimeSeries transform(TimeSeries timeSeries, AnomalyDetectionContext anomalyDetectionContext) {
    Interval timeSeriesInterval = timeSeries.getTimeSeriesInterval();
    long startTime = timeSeriesInterval.getStartMillis();
    long endTime = timeSeriesInterval.getEndMillis();
    long bucketSizeInMillis = anomalyDetectionContext.getBucketSizeInMS();
    int movingAverageWindowSize = Integer.valueOf(getProperties().getProperty(MOVING_AVERAGE_SMOOTHING_WINDOW_SIZE));
    // Check if the moving average window size is larger than the time series itself
    long transformedStartTime = startTime + bucketSizeInMillis * (movingAverageWindowSize - 1);
    if (transformedStartTime > endTime) {
        String metricName = anomalyDetectionContext.getAnomalyDetectionFunction().getSpec().getTopicMetric();
        DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
        LOGGER.warn("Input time series (Metric:{}, Dimension:{}) is shorter than the moving average " + "smoothing window; therefore, smoothing is not applied on this time series.", metricName, dimensionMap);
        return timeSeries;
    }
    TimeSeries transformedTimeSeries = new TimeSeries();
    Interval transformedInterval = new Interval(transformedStartTime, endTime);
    transformedTimeSeries.setTimeSeriesInterval(transformedInterval);
    for (long timeKeyToTransform : timeSeries.timestampSet()) {
        if (!transformedInterval.contains(timeKeyToTransform)) {
            continue;
        }
        double sum = 0d;
        int count = 0;
        for (int i = 0; i < movingAverageWindowSize; ++i) {
            long timeKey = timeKeyToTransform - bucketSizeInMillis * i;
            if (timeSeries.hasTimestamp(timeKey)) {
                sum += timeSeries.get(timeKey);
                ++count;
            }
        }
        // count is at least one due to the existence of timeKeyToTransform
        double average = sum / count;
        transformedTimeSeries.set(timeKeyToTransform, average);
    }
    return transformedTimeSeries;
}

Example 10 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class ZeroRemovalFunction method transform.

/**
   * Removes value 0.0 from the time series. The reason to apply this transformation function is
   * that ThirdEye currently returns empty values as 0.0. Therefore, we need to remove those values.
   *
   * @param timeSeries the time series that provides the data points to be transformed.
   * @param anomalyDetectionContext the anomaly detection context that could provide additional
   *                                information for the transformation.
   * @return a time series that have value 0.0 removed.
   */
@Override
public TimeSeries transform(TimeSeries timeSeries, AnomalyDetectionContext anomalyDetectionContext) {
    TimeSeries transformedTimeSeries = new TimeSeries();
    Interval timeSeriesInterval = timeSeries.getTimeSeriesInterval();
    transformedTimeSeries.setTimeSeriesInterval(timeSeriesInterval);
    for (long timestamp : timeSeries.timestampSet()) {
        double value = timeSeries.get(timestamp);
        if (value != 0d && timeSeriesInterval.contains(timestamp)) {
            transformedTimeSeries.set(timestamp, value);
        }
    }
    return transformedTimeSeries;
}