Search in sources :

Example 6 with DimensionKey

use of com.linkedin.thirdeye.api.DimensionKey in project pinot by linkedin.

the class DetectionTaskRunner method fetchData.

private AnomalyDetectionInputContext fetchData(DateTime windowStart, DateTime windowEnd) throws JobExecutionException, ExecutionException {
    AnomalyDetectionInputContext adContext = new AnomalyDetectionInputContext();
    // Get Time Series
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis());
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    Map<DimensionMap, MetricTimeSeries> dimensionMapMetricTimeSeriesMap = new HashMap<>();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        // If the current time series belongs to OTHER dimension, which consists of time series whose
        // sum of all its values belows 1% of sum of all time series values, then its anomaly is
        // meaningless and hence we don't want to detection anomalies on it.
        String[] dimensionValues = dimensionKey.getDimensionValues();
        boolean isOTHERDimension = false;
        for (String dimensionValue : dimensionValues) {
            if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER) || dimensionValue.equalsIgnoreCase(ResponseParserUtils.UNKNOWN)) {
                isOTHERDimension = true;
                break;
            }
        }
        if (isOTHERDimension) {
            continue;
        }
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        dimensionMapMetricTimeSeriesMap.put(dimensionMap, entry.getValue());
        if (entry.getValue().getTimeWindowSet().size() < 1) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
    }
    adContext.setDimensionKeyMetricTimeSeriesMap(dimensionMapMetricTimeSeriesMap);
    // Get existing anomalies for this time range and this function id for all combinations of dimensions
    List<MergedAnomalyResultDTO> knownMergedAnomalies;
    if (anomalyFunction.useHistoryAnomaly()) {
        // if this anomaly function uses history data, then we get all time ranges
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    } else {
        // otherwise, we only get the merge anomaly for current window in order to remove duplicate raw anomalies
        List<Pair<Long, Long>> currentTimeRange = new ArrayList<>();
        currentTimeRange.add(new Pair<>(windowStart.getMillis(), windowEnd.getMillis()));
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), currentTimeRange);
    }
    // Sort the known merged and raw anomalies by their dimension names
    ArrayListMultimap<DimensionMap, MergedAnomalyResultDTO> dimensionMapToKnownMergedAnomalies = ArrayListMultimap.create();
    for (MergedAnomalyResultDTO knownMergedAnomaly : knownMergedAnomalies) {
        dimensionMapToKnownMergedAnomalies.put(knownMergedAnomaly.getDimensions(), knownMergedAnomaly);
    }
    adContext.setKnownMergedAnomalies(dimensionMapToKnownMergedAnomalies);
    // We always find existing raw anomalies to prevent duplicate raw anomalies are generated
    List<RawAnomalyResultDTO> existingRawAnomalies = getExistingRawAnomalies(anomalyFunctionSpec.getId(), windowStart.getMillis(), windowEnd.getMillis());
    ArrayListMultimap<DimensionMap, RawAnomalyResultDTO> dimensionNamesToKnownRawAnomalies = ArrayListMultimap.create();
    for (RawAnomalyResultDTO existingRawAnomaly : existingRawAnomalies) {
        dimensionNamesToKnownRawAnomalies.put(existingRawAnomaly.getDimensions(), existingRawAnomaly);
    }
    adContext.setExistingRawAnomalies(dimensionNamesToKnownRawAnomalies);
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(DAO_REGISTRY.getOverrideConfigDAO(), anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    adContext.setScalingFactors(scalingFactors);
    return adContext;
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) MetricTimeSeries(com.linkedin.thirdeye.api.MetricTimeSeries) ScalingFactor(com.linkedin.thirdeye.detector.metric.transfer.ScalingFactor) RawAnomalyResultDTO(com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO) DimensionKey(com.linkedin.thirdeye.api.DimensionKey) MergedAnomalyResultDTO(com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO) DimensionMap(com.linkedin.thirdeye.api.DimensionMap) HashMap(java.util.HashMap) Map(java.util.Map) DimensionMap(com.linkedin.thirdeye.api.DimensionMap) Pair(com.linkedin.pinot.pql.parsers.utils.Pair)

Aggregations

DimensionKey (com.linkedin.thirdeye.api.DimensionKey)6 MetricTimeSeries (com.linkedin.thirdeye.api.MetricTimeSeries)6 DimensionMap (com.linkedin.thirdeye.api.DimensionMap)3 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 Map (java.util.Map)3 Pair (com.linkedin.pinot.pql.parsers.utils.Pair)2 TimeSeriesResponse (com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse)2 RawAnomalyResultDTO (com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO)2 ExecutionException (java.util.concurrent.ExecutionException)2 JobExecutionException (org.quartz.JobExecutionException)2 MetricSchema (com.linkedin.thirdeye.api.MetricSchema)1 MetricType (com.linkedin.thirdeye.api.MetricType)1 TimeGranularity (com.linkedin.thirdeye.api.TimeGranularity)1 TimeSeriesMetric (com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric)1 MergedAnomalyResultDTO (com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO)1 BaseAnomalyFunction (com.linkedin.thirdeye.detector.function.BaseAnomalyFunction)1 ScalingFactor (com.linkedin.thirdeye.detector.metric.transfer.ScalingFactor)1 IOException (java.io.IOException)1 InvocationTargetException (java.lang.reflect.InvocationTargetException)1