Examples with TimeSeries com.linkedin.thirdeye.anomalydetection.context.TimeSeries used on opensource projects

Example 1 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class BackwardAnomalyFunctionUtils method splitSetsOfTimeSeries.

/**
   * Splits a MetricTimeSeries to current (observed) time series and baselines. The list of time
   * series is sorted by the start time of their interval in the reversed natural order. Therefore,
   * the current time series is located at the beginning of the returned list.
   *
   * @param metricTimeSeries the metric time series that contains current and baseline time series.
   * @param metricName the metric name to retrieve the value from the given metric time series.
   * @param timeSeriesIntervals the intervals of the split time series.
   * @return a list of time series, which are split from the metric time series.
   */
public static List<TimeSeries> splitSetsOfTimeSeries(MetricTimeSeries metricTimeSeries, String metricName, List<Interval> timeSeriesIntervals) {
    List<TimeSeries> timeSeriesList = new ArrayList<>(timeSeriesIntervals.size());
    for (Interval interval : timeSeriesIntervals) {
        TimeSeries timeSeries = new TimeSeries();
        timeSeries.setTimeSeriesInterval(interval);
        timeSeriesList.add(timeSeries);
    }
    // Sort time series by their start time in reversed natural order, i.e., the latest time series
    // is arranged in the front of the list
    Collections.sort(timeSeriesList, new TimeSeriesStartTimeComparator().reversed());
    // the timestamp and its value could be inserted to multiple time series.
    for (long timestamp : metricTimeSeries.getTimeWindowSet()) {
        for (TimeSeries timeSeries : timeSeriesList) {
            if (timeSeries.getTimeSeriesInterval().contains(timestamp)) {
                double value = metricTimeSeries.get(timestamp, metricName).doubleValue();
                timeSeries.set(timestamp, value);
            }
        }
    }
    return timeSeriesList;
}

Example 2 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class BackwardAnomalyFunctionUtils method buildAnomalyDetectionContext.

/**
   * Returns an anomaly detection context from the given information.
   *
   * @param anomalyFunction the anomaly function for anomaly detection.
   * @param timeSeries the given time series.
   * @param metric the metric name of the given time series.
   * @param exploredDimensions the dimension map of the given time series.
   * @param windowStart the start of the interval of the time series.
   * @param windowEnd the end of the interval of the time series.
   *
   * @return an anomaly detection context from the given information.
   */
public static AnomalyDetectionContext buildAnomalyDetectionContext(AnomalyDetectionFunction anomalyFunction, MetricTimeSeries timeSeries, String metric, DimensionMap exploredDimensions, int bucketSize, TimeUnit bucketUnit, DateTime windowStart, DateTime windowEnd) {
    // Create the anomaly detection context for the new modularized anomaly function
    AnomalyDetectionContext anomalyDetectionContext = new AnomalyDetectionContext();
    anomalyDetectionContext.setBucketSizeInMS(AnomalyDetectionUtils.getBucketInMillis(bucketSize, bucketUnit));
    anomalyDetectionContext.setAnomalyDetectionFunction(anomalyFunction);
    // Construct TimeSeriesKey
    TimeSeriesKey timeSeriesKey = new TimeSeriesKey();
    timeSeriesKey.setDimensionMap(exploredDimensions);
    timeSeriesKey.setMetricName(metric);
    anomalyDetectionContext.setTimeSeriesKey(timeSeriesKey);
    // Split time series to observed time series and baselines for each metric
    for (String metricName : anomalyFunction.getSpec().getMetrics()) {
        List<Interval> intervals = anomalyFunction.getTimeSeriesIntervals(windowStart.getMillis(), windowEnd.getMillis());
        List<TimeSeries> timeSeriesList = BackwardAnomalyFunctionUtils.splitSetsOfTimeSeries(timeSeries, metricName, intervals);
        anomalyDetectionContext.setCurrent(metricName, timeSeriesList.get(0));
        timeSeriesList.remove(0);
        anomalyDetectionContext.setBaselines(metricName, timeSeriesList);
    }
    return anomalyDetectionContext;
}

Example 3 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class MinMaxThresholdDetectionModel method detect.

@Override
public List<RawAnomalyResultDTO> detect(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    // Get min / max props
    Double min = null;
    if (properties.containsKey(MIN_VAL)) {
        min = Double.valueOf(properties.getProperty(MIN_VAL));
    }
    Double max = null;
    if (properties.containsKey(MAX_VAL)) {
        max = Double.valueOf(properties.getProperty(MAX_VAL));
    }
    TimeSeries timeSeries = anomalyDetectionContext.getTransformedCurrent(metricName);
    // Compute the weight of this time series (average across whole)
    double averageValue = 0;
    for (long time : timeSeries.timestampSet()) {
        averageValue += timeSeries.get(time);
    }
    // Compute the bucket size, so we can iterate in those steps
    long bucketMillis = anomalyDetectionContext.getBucketSizeInMS();
    Interval timeSeriesInterval = timeSeries.getTimeSeriesInterval();
    long numBuckets = Math.abs(timeSeriesInterval.getEndMillis() - timeSeriesInterval.getStartMillis()) / bucketMillis;
    // avg value of this time series
    averageValue /= numBuckets;
    DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
    for (long timeBucket : timeSeries.timestampSet()) {
        double value = timeSeries.get(timeBucket);
        double deviationFromThreshold = getDeviationFromThreshold(value, min, max);
        if (deviationFromThreshold != 0) {
            RawAnomalyResultDTO anomalyResult = new RawAnomalyResultDTO();
            anomalyResult.setProperties(properties.toString());
            anomalyResult.setStartTime(timeBucket);
            // point-in-time
            anomalyResult.setEndTime(timeBucket + bucketMillis);
            anomalyResult.setDimensions(dimensionMap);
            anomalyResult.setScore(averageValue);
            // higher change, higher the severity
            anomalyResult.setWeight(deviationFromThreshold);
            anomalyResult.setAvgCurrentVal(value);
            String message = String.format(DEFAULT_MESSAGE_TEMPLATE, deviationFromThreshold, value, min, max);
            anomalyResult.setMessage(message);
            if (value == 0.0) {
                anomalyResult.setDataMissing(true);
            }
            anomalyResults.add(anomalyResult);
        }
    }
    return anomalyResults;
}

Example 4 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class SimpleThresholdDetectionModel method detect.

@Override
public List<RawAnomalyResultDTO> detect(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    // Get thresholds
    double changeThreshold = Double.valueOf(getProperties().getProperty(CHANGE_THRESHOLD));
    double volumeThreshold = 0d;
    if (getProperties().containsKey(AVERAGE_VOLUME_THRESHOLD)) {
        volumeThreshold = Double.valueOf(getProperties().getProperty(AVERAGE_VOLUME_THRESHOLD));
    }
    long bucketSizeInMillis = anomalyDetectionContext.getBucketSizeInMS();
    // Compute the weight of this time series (average across whole)
    TimeSeries currentTimeSeries = anomalyDetectionContext.getTransformedCurrent(metricName);
    double averageValue = 0;
    for (long time : currentTimeSeries.timestampSet()) {
        averageValue += currentTimeSeries.get(time);
    }
    Interval currentInterval = currentTimeSeries.getTimeSeriesInterval();
    long currentStart = currentInterval.getStartMillis();
    long currentEnd = currentInterval.getEndMillis();
    long numBuckets = (currentEnd - currentStart) / bucketSizeInMillis;
    if (numBuckets != 0) {
        averageValue /= numBuckets;
    }
    // Check if this time series even meets our volume threshold
    DimensionMap dimensionMap = anomalyDetectionContext.getTimeSeriesKey().getDimensionMap();
    if (averageValue < volumeThreshold) {
        LOGGER.info("{} does not meet volume threshold {}: {}", dimensionMap, volumeThreshold, averageValue);
        // empty list
        return anomalyResults;
    }
    PredictionModel predictionModel = anomalyDetectionContext.getTrainedPredictionModel(metricName);
    if (!(predictionModel instanceof ExpectedTimeSeriesPredictionModel)) {
        LOGGER.info("SimpleThresholdDetectionModel detection model expects an ExpectedTimeSeriesPredictionModel but the trained prediction model in anomaly detection context is not.");
        // empty list
        return anomalyResults;
    }
    ExpectedTimeSeriesPredictionModel expectedTimeSeriesPredictionModel = (ExpectedTimeSeriesPredictionModel) predictionModel;
    TimeSeries expectedTimeSeries = expectedTimeSeriesPredictionModel.getExpectedTimeSeries();
    Interval expectedTSInterval = expectedTimeSeries.getTimeSeriesInterval();
    long expectedStart = expectedTSInterval.getStartMillis();
    long seasonalOffset = currentStart - expectedStart;
    for (long currentTimestamp : currentTimeSeries.timestampSet()) {
        long expectedTimestamp = currentTimestamp - seasonalOffset;
        if (!expectedTimeSeries.hasTimestamp(expectedTimestamp)) {
            continue;
        }
        double baselineValue = expectedTimeSeries.get(expectedTimestamp);
        double currentValue = currentTimeSeries.get(currentTimestamp);
        if (isAnomaly(currentValue, baselineValue, changeThreshold)) {
            RawAnomalyResultDTO anomalyResult = new RawAnomalyResultDTO();
            anomalyResult.setDimensions(dimensionMap);
            anomalyResult.setProperties(getProperties().toString());
            anomalyResult.setStartTime(currentTimestamp);
            // point-in-time
            anomalyResult.setEndTime(currentTimestamp + bucketSizeInMillis);
            anomalyResult.setScore(averageValue);
            anomalyResult.setWeight(calculateChange(currentValue, baselineValue));
            anomalyResult.setAvgCurrentVal(currentValue);
            anomalyResult.setAvgBaselineVal(baselineValue);
            String message = getAnomalyResultMessage(changeThreshold, currentValue, baselineValue);
            anomalyResult.setMessage(message);
            anomalyResults.add(anomalyResult);
            if (currentValue == 0.0 || baselineValue == 0.0) {
                anomalyResult.setDataMissing(true);
            }
        }
    }
    return anomalyResults;
}

Example 5 with TimeSeries

use of com.linkedin.thirdeye.anomalydetection.context.TimeSeries in project pinot by linkedin.

the class AbstractModularizedAnomalyFunction method transformTimeSeries.

/**
   * Transform the current time series and baselines.
   *
   * TODO: Apply Chain-of-Responsibility on the transformation chain
   *
   * @param metricName the name of the metric on which we apply transformation and prediction
   * @param anomalyDetectionContext anomaly detection context that contains the time series to be
   *                                transformed.
   */
private void transformTimeSeries(String metricName, AnomalyDetectionContext anomalyDetectionContext) {
    // Transform the observed (current) time series
    if (anomalyDetectionContext.getTransformedCurrent(metricName) == null) {
        anomalyDetectionContext.setTransformedCurrent(metricName, anomalyDetectionContext.getCurrent(metricName));
    }
    List<TransformationFunction> currentTimeSeriesTransformationChain = getCurrentTimeSeriesTransformationChain();
    if (CollectionUtils.isNotEmpty(currentTimeSeriesTransformationChain)) {
        for (TransformationFunction tf : currentTimeSeriesTransformationChain) {
            anomalyDetectionContext.setTransformedCurrent(metricName, tf.transform(anomalyDetectionContext.getTransformedCurrent(metricName), anomalyDetectionContext));
        }
    }
    // Transform baseline time series
    if (anomalyDetectionContext.getTransformedBaselines(metricName) == null) {
        anomalyDetectionContext.setTransformedBaselines(metricName, anomalyDetectionContext.getBaselines(metricName));
    }
    List<TransformationFunction> baselineTimeSeriesTransformationChain = getBaselineTimeSeriesTransformationChain();
    if (CollectionUtils.isNotEmpty(anomalyDetectionContext.getTransformedBaselines(metricName)) && CollectionUtils.isNotEmpty(baselineTimeSeriesTransformationChain)) {
        for (TransformationFunction tf : baselineTimeSeriesTransformationChain) {
            List<TimeSeries> transformedBaselines = new ArrayList<>();
            for (TimeSeries ts : anomalyDetectionContext.getTransformedBaselines(metricName)) {
                TimeSeries transformedTS = tf.transform(ts, anomalyDetectionContext);
                transformedBaselines.add(transformedTS);
            }
            anomalyDetectionContext.setTransformedBaselines(metricName, transformedBaselines);
        }
    }
}