Examples with RawAnomalyResultDTO com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO used on opensource projects

Example 46 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class AnomalyMergeExecutor method performMergeBasedOnFunctionIdAndDimensions.

private void performMergeBasedOnFunctionIdAndDimensions(AnomalyFunctionDTO function, AnomalyMergeConfig mergeConfig, List<RawAnomalyResultDTO> unmergedResults, List<MergedAnomalyResultDTO> output) {
    Map<DimensionMap, List<RawAnomalyResultDTO>> dimensionsResultMap = new HashMap<>();
    for (RawAnomalyResultDTO anomalyResult : unmergedResults) {
        DimensionMap exploredDimensions = anomalyResult.getDimensions();
        if (!dimensionsResultMap.containsKey(exploredDimensions)) {
            dimensionsResultMap.put(exploredDimensions, new ArrayList<>());
        }
        dimensionsResultMap.get(exploredDimensions).add(anomalyResult);
    }
    for (DimensionMap exploredDimensions : dimensionsResultMap.keySet()) {
        List<RawAnomalyResultDTO> unmergedResultsByDimensions = dimensionsResultMap.get(exploredDimensions);
        long anomalyWindowStart = Long.MAX_VALUE;
        long anomalyWindowEnd = Long.MIN_VALUE;
        for (RawAnomalyResultDTO unmergedResultsByDimension : unmergedResultsByDimensions) {
            anomalyWindowStart = Math.min(anomalyWindowStart, unmergedResultsByDimension.getStartTime());
            anomalyWindowEnd = Math.max(anomalyWindowEnd, unmergedResultsByDimension.getEndTime());
        }
        // NOTE: We get "latest overlapped (Conflict)" merged anomaly instead of "latest" merged anomaly in order to
        // prevent the merge results of current (online) detection interfere the merge results of back-fill (offline)
        // detection.
        // Moreover, the window start is modified by mergeConfig.getSequentialAllowedGap() in order to allow a gap between
        // anomalies to be merged.
        MergedAnomalyResultDTO latestOverlappedMergedResult = mergedResultDAO.findLatestConflictByFunctionIdDimensions(function.getId(), exploredDimensions.toString(), anomalyWindowStart - mergeConfig.getSequentialAllowedGap(), anomalyWindowEnd);
        List<MergedAnomalyResultDTO> mergedResults = AnomalyTimeBasedSummarizer.mergeAnomalies(latestOverlappedMergedResult, unmergedResultsByDimensions, mergeConfig.getMaxMergeDurationLength(), mergeConfig.getSequentialAllowedGap());
        for (MergedAnomalyResultDTO mergedResult : mergedResults) {
            mergedResult.setFunction(function);
            mergedResult.setDimensions(exploredDimensions);
        }
        LOG.info("Merging [{}] raw anomalies into [{}] merged anomalies for function id : [{}] and dimensions : [{}]", unmergedResultsByDimensions.size(), mergedResults.size(), function.getId(), exploredDimensions);
        output.addAll(mergedResults);
    }
}

Example 47 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class DetectionTaskRunner method handleResults.

private void handleResults(List<RawAnomalyResultDTO> results) {
    for (RawAnomalyResultDTO result : results) {
        try {
            // Properties that always come from the function spec
            AnomalyFunctionDTO spec = anomalyFunction.getSpec();
            // make sure score and weight are valid numbers
            result.setScore(normalize(result.getScore()));
            result.setWeight(normalize(result.getWeight()));
            result.setFunction(spec);
        } catch (Exception e) {
            LOG.error("Exception in saving anomaly result : " + result.toString(), e);
        }
    }
}

Example 48 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class DetectionTaskRunner method fetchData.

private AnomalyDetectionInputContext fetchData(DateTime windowStart, DateTime windowEnd) throws JobExecutionException, ExecutionException {
    AnomalyDetectionInputContext adContext = new AnomalyDetectionInputContext();
    // Get Time Series
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis());
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    Map<DimensionMap, MetricTimeSeries> dimensionMapMetricTimeSeriesMap = new HashMap<>();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        // If the current time series belongs to OTHER dimension, which consists of time series whose
        // sum of all its values belows 1% of sum of all time series values, then its anomaly is
        // meaningless and hence we don't want to detection anomalies on it.
        String[] dimensionValues = dimensionKey.getDimensionValues();
        boolean isOTHERDimension = false;
        for (String dimensionValue : dimensionValues) {
            if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER) || dimensionValue.equalsIgnoreCase(ResponseParserUtils.UNKNOWN)) {
                isOTHERDimension = true;
                break;
            }
        }
        if (isOTHERDimension) {
            continue;
        }
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        dimensionMapMetricTimeSeriesMap.put(dimensionMap, entry.getValue());
        if (entry.getValue().getTimeWindowSet().size() < 1) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
    }
    adContext.setDimensionKeyMetricTimeSeriesMap(dimensionMapMetricTimeSeriesMap);
    // Get existing anomalies for this time range and this function id for all combinations of dimensions
    List<MergedAnomalyResultDTO> knownMergedAnomalies;
    if (anomalyFunction.useHistoryAnomaly()) {
        // if this anomaly function uses history data, then we get all time ranges
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    } else {
        // otherwise, we only get the merge anomaly for current window in order to remove duplicate raw anomalies
        List<Pair<Long, Long>> currentTimeRange = new ArrayList<>();
        currentTimeRange.add(new Pair<>(windowStart.getMillis(), windowEnd.getMillis()));
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), currentTimeRange);
    }
    // Sort the known merged and raw anomalies by their dimension names
    ArrayListMultimap<DimensionMap, MergedAnomalyResultDTO> dimensionMapToKnownMergedAnomalies = ArrayListMultimap.create();
    for (MergedAnomalyResultDTO knownMergedAnomaly : knownMergedAnomalies) {
        dimensionMapToKnownMergedAnomalies.put(knownMergedAnomaly.getDimensions(), knownMergedAnomaly);
    }
    adContext.setKnownMergedAnomalies(dimensionMapToKnownMergedAnomalies);
    // We always find existing raw anomalies to prevent duplicate raw anomalies are generated
    List<RawAnomalyResultDTO> existingRawAnomalies = getExistingRawAnomalies(anomalyFunctionSpec.getId(), windowStart.getMillis(), windowEnd.getMillis());
    ArrayListMultimap<DimensionMap, RawAnomalyResultDTO> dimensionNamesToKnownRawAnomalies = ArrayListMultimap.create();
    for (RawAnomalyResultDTO existingRawAnomaly : existingRawAnomalies) {
        dimensionNamesToKnownRawAnomalies.put(existingRawAnomaly.getDimensions(), existingRawAnomaly);
    }
    adContext.setExistingRawAnomalies(dimensionNamesToKnownRawAnomalies);
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(DAO_REGISTRY.getOverrideConfigDAO(), anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    adContext.setScalingFactors(scalingFactors);
    return adContext;
}