Examples with MetricTimeSeries com.linkedin.thirdeye.api.MetricTimeSeries used on opensource projects

Example 6 with MetricTimeSeries

use of com.linkedin.thirdeye.api.MetricTimeSeries in project pinot by linkedin.

the class TimeSeriesUtil method extractMetricTimeSeriesByDimension.

/**
   * Extract current and baseline values from the parsed Pinot results. There are two possible time series for presenting
   * the time series after anomaly detection: 1. the time series with a specific dimension and 2. the time series for
   * OTHER dimension.
   *
   * For case 1, the input map should contain only one time series and hence we can just return it. For case 2, the
   * input map would contain all combination of explored dimension and hence we need to filter out the one for OTHER
   * dimension.
   *
   * @param metricTimeSeriesMap
   *
   * @return the time series when the anomaly is detected
   */
private static MetricTimeSeries extractMetricTimeSeriesByDimension(Map<DimensionKey, MetricTimeSeries> metricTimeSeriesMap) {
    MetricTimeSeries metricTimeSeries = null;
    if (MapUtils.isNotEmpty(metricTimeSeriesMap)) {
        // Therefore, we need to get the time series of OTHER dimension manually.
        if (metricTimeSeriesMap.size() == 1) {
            Iterator<MetricTimeSeries> ite = metricTimeSeriesMap.values().iterator();
            if (ite.hasNext()) {
                metricTimeSeries = ite.next();
            }
        } else {
            // Retrieve the time series of OTHER dimension
            Iterator<Map.Entry<DimensionKey, MetricTimeSeries>> ite = metricTimeSeriesMap.entrySet().iterator();
            while (ite.hasNext()) {
                Map.Entry<DimensionKey, MetricTimeSeries> entry = ite.next();
                DimensionKey dimensionKey = entry.getKey();
                boolean foundOTHER = false;
                for (String dimensionValue : dimensionKey.getDimensionValues()) {
                    if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER)) {
                        metricTimeSeries = entry.getValue();
                        foundOTHER = true;
                        break;
                    }
                }
                if (foundOTHER) {
                    break;
                }
            }
        }
    }
    return metricTimeSeries;
}

Example 7 with MetricTimeSeries

use of com.linkedin.thirdeye.api.MetricTimeSeries in project pinot by linkedin.

the class BackwardAnomalyFunctionUtils method splitSetsOfTimeSeries.

/**
   * Splits a MetricTimeSeries to current (observed) time series and baselines. The list of time
   * series is sorted by the start time of their interval in the reversed natural order. Therefore,
   * the current time series is located at the beginning of the returned list.
   *
   * @param metricTimeSeries the metric time series that contains current and baseline time series.
   * @param metricName the metric name to retrieve the value from the given metric time series.
   * @param timeSeriesIntervals the intervals of the split time series.
   * @return a list of time series, which are split from the metric time series.
   */
public static List<TimeSeries> splitSetsOfTimeSeries(MetricTimeSeries metricTimeSeries, String metricName, List<Interval> timeSeriesIntervals) {
    List<TimeSeries> timeSeriesList = new ArrayList<>(timeSeriesIntervals.size());
    for (Interval interval : timeSeriesIntervals) {
        TimeSeries timeSeries = new TimeSeries();
        timeSeries.setTimeSeriesInterval(interval);
        timeSeriesList.add(timeSeries);
    }
    // Sort time series by their start time in reversed natural order, i.e., the latest time series
    // is arranged in the front of the list
    Collections.sort(timeSeriesList, new TimeSeriesStartTimeComparator().reversed());
    // the timestamp and its value could be inserted to multiple time series.
    for (long timestamp : metricTimeSeries.getTimeWindowSet()) {
        for (TimeSeries timeSeries : timeSeriesList) {
            if (timeSeries.getTimeSeriesInterval().contains(timestamp)) {
                double value = metricTimeSeries.get(timestamp, metricName).doubleValue();
                timeSeries.set(timestamp, value);
            }
        }
    }
    return timeSeriesList;
}

Example 8 with MetricTimeSeries

use of com.linkedin.thirdeye.api.MetricTimeSeries in project pinot by linkedin.

the class BackwardAnomalyFunctionUtils method buildAnomalyDetectionContext.

/**
   * Returns an anomaly detection context from the given information.
   *
   * @param anomalyFunction the anomaly function for anomaly detection.
   * @param timeSeries the given time series.
   * @param metric the metric name of the given time series.
   * @param exploredDimensions the dimension map of the given time series.
   * @param windowStart the start of the interval of the time series.
   * @param windowEnd the end of the interval of the time series.
   *
   * @return an anomaly detection context from the given information.
   */
public static AnomalyDetectionContext buildAnomalyDetectionContext(AnomalyDetectionFunction anomalyFunction, MetricTimeSeries timeSeries, String metric, DimensionMap exploredDimensions, int bucketSize, TimeUnit bucketUnit, DateTime windowStart, DateTime windowEnd) {
    // Create the anomaly detection context for the new modularized anomaly function
    AnomalyDetectionContext anomalyDetectionContext = new AnomalyDetectionContext();
    anomalyDetectionContext.setBucketSizeInMS(AnomalyDetectionUtils.getBucketInMillis(bucketSize, bucketUnit));
    anomalyDetectionContext.setAnomalyDetectionFunction(anomalyFunction);
    // Construct TimeSeriesKey
    TimeSeriesKey timeSeriesKey = new TimeSeriesKey();
    timeSeriesKey.setDimensionMap(exploredDimensions);
    timeSeriesKey.setMetricName(metric);
    anomalyDetectionContext.setTimeSeriesKey(timeSeriesKey);
    // Split time series to observed time series and baselines for each metric
    for (String metricName : anomalyFunction.getSpec().getMetrics()) {
        List<Interval> intervals = anomalyFunction.getTimeSeriesIntervals(windowStart.getMillis(), windowEnd.getMillis());
        List<TimeSeries> timeSeriesList = BackwardAnomalyFunctionUtils.splitSetsOfTimeSeries(timeSeries, metricName, intervals);
        anomalyDetectionContext.setCurrent(metricName, timeSeriesList.get(0));
        timeSeriesList.remove(0);
        anomalyDetectionContext.setBaselines(metricName, timeSeriesList);
    }
    return anomalyDetectionContext;
}

Example 9 with MetricTimeSeries

use of com.linkedin.thirdeye.api.MetricTimeSeries in project pinot by linkedin.

the class AbstractModularizedAnomalyFunction method getTimeSeriesView.

// TODO: Generate time series view using ViewModel
@Override
public AnomalyTimelinesView getTimeSeriesView(MetricTimeSeries timeSeries, long bucketMillis, String metric, long viewWindowStartTime, long viewWindowEndTime, List<MergedAnomalyResultDTO> knownAnomalies) {
    AnomalyDetectionContext anomalyDetectionContext = BackwardAnomalyFunctionUtils.buildAnomalyDetectionContext(this, timeSeries, spec.getTopicMetric(), null, spec.getBucketSize(), spec.getBucketUnit(), new DateTime(viewWindowStartTime), new DateTime(viewWindowEndTime));
    String mainMetric = anomalyDetectionContext.getAnomalyDetectionFunction().getSpec().getTopicMetric();
    this.transformAndPredictTimeSeries(mainMetric, anomalyDetectionContext);
    TimeSeries observedTS = anomalyDetectionContext.getTransformedCurrent(mainMetric);
    TimeSeries expectedTS = ((ExpectedTimeSeriesPredictionModel) anomalyDetectionContext.getTrainedPredictionModel(mainMetric)).getExpectedTimeSeries();
    long expectedTSStartTime = expectedTS.getTimeSeriesInterval().getStartMillis();
    // Construct AnomalyTimelinesView
    AnomalyTimelinesView anomalyTimelinesView = new AnomalyTimelinesView();
    int bucketCount = (int) ((viewWindowEndTime - viewWindowStartTime) / bucketMillis);
    for (int i = 0; i < bucketCount; ++i) {
        long currentBucketMillis = viewWindowStartTime + i * bucketMillis;
        long baselineBucketMillis = expectedTSStartTime + i * bucketMillis;
        double observedValue = 0d;
        if (observedTS.hasTimestamp(currentBucketMillis)) {
            observedValue = observedTS.get(currentBucketMillis);
        }
        double expectedValue = 0d;
        if (expectedTS.hasTimestamp(baselineBucketMillis)) {
            expectedValue = expectedTS.get(baselineBucketMillis);
        }
        TimeBucket timebucket = new TimeBucket(currentBucketMillis, currentBucketMillis + bucketMillis, baselineBucketMillis, baselineBucketMillis + bucketMillis);
        anomalyTimelinesView.addTimeBuckets(timebucket);
        anomalyTimelinesView.addCurrentValues(observedValue);
        anomalyTimelinesView.addBaselineValues(expectedValue);
    }
    return anomalyTimelinesView;
}

Example 10 with MetricTimeSeries

use of com.linkedin.thirdeye.api.MetricTimeSeries in project pinot by linkedin.

the class AnomalyFunctionResource method analyze.

@POST
@Path("/analyze")
@Consumes(MediaType.APPLICATION_JSON)
public Response analyze(AnomalyFunctionDTO anomalyFunctionSpec, @QueryParam("startTime") Long startTime, @QueryParam("endTime") Long endTime) throws Exception {
    // TODO: replace this with Job/Task framework and job tracker page
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(startTime, endTime);
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>();
    List<RawAnomalyResultDTO> results = new ArrayList<>();
    List<String> collectionDimensions = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(anomalyFunctionSpec.getCollection()).getDimensions();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        if (entry.getValue().getTimeWindowSet().size() < 2) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
        try {
            // Run algorithm
            MetricTimeSeries metricTimeSeries = entry.getValue();
            LOG.info("Analyzing anomaly function with dimensionKey: {}, windowStart: {}, windowEnd: {}", dimensionMap, startTime, endTime);
            List<RawAnomalyResultDTO> resultsOfAnEntry = anomalyFunction.analyze(dimensionMap, metricTimeSeries, new DateTime(startTime), new DateTime(endTime), new ArrayList<>());
            if (resultsOfAnEntry.size() != 0) {
                results.addAll(resultsOfAnEntry);
            }
            LOG.info("{} has {} anomalies in window {} to {}", dimensionMap, resultsOfAnEntry.size(), new DateTime(startTime), new DateTime(endTime));
        } catch (Exception e) {
            LOG.error("Could not compute for {}", dimensionMap, e);
        }
    }
    if (results.size() > 0) {
        List<RawAnomalyResultDTO> validResults = new ArrayList<>();
        for (RawAnomalyResultDTO anomaly : results) {
            if (!anomaly.isDataMissing()) {
                LOG.info("Found anomaly, sev [{}] start [{}] end [{}]", anomaly.getWeight(), new DateTime(anomaly.getStartTime()), new DateTime(anomaly.getEndTime()));
                validResults.add(anomaly);
            }
        }
        anomalyResults.addAll(validResults);
    }
    return Response.ok(anomalyResults).build();
}