Examples with DimensionMap com.linkedin.thirdeye.api.DimensionMap used on opensource projects

Example 16 with DimensionMap

use of com.linkedin.thirdeye.api.DimensionMap in project pinot by linkedin.

the class TestMinMaxThresholdFunction method timeSeriesDataProvider.

@DataProvider(name = "timeSeriesDataProvider")
public Object[][] timeSeriesDataProvider() {
    // The properties for the testing time series
    Properties properties = new Properties();
    long bucketSizeInMS = TimeUnit.SECONDS.toMillis(1);
    // Set up time series key for the testing time series
    TimeSeriesKey timeSeriesKey = new TimeSeriesKey();
    String metric = mainMetric;
    timeSeriesKey.setMetricName(metric);
    DimensionMap dimensionMap = new DimensionMap();
    dimensionMap.put("dimensionName1", "dimensionValue1");
    dimensionMap.put("dimensionName2", "dimensionValue2");
    timeSeriesKey.setDimensionMap(dimensionMap);
    TimeSeries observedTimeSeries = new TimeSeries();
    {
        observedTimeSeries.set(observedStartTime, 10d);
        observedTimeSeries.set(observedStartTime + bucketMillis, 15d);
        observedTimeSeries.set(observedStartTime + bucketMillis * 2, 13d);
        observedTimeSeries.set(observedStartTime + bucketMillis * 3, 22d);
        observedTimeSeries.set(observedStartTime + bucketMillis * 4, 8d);
        Interval observedTimeSeriesInterval = new Interval(observedStartTime, observedStartTime + bucketMillis * 5);
        observedTimeSeries.setTimeSeriesInterval(observedTimeSeriesInterval);
    }
    return new Object[][] { { properties, timeSeriesKey, bucketSizeInMS, observedTimeSeries } };
}

Example 17 with DimensionMap

use of com.linkedin.thirdeye.api.DimensionMap in project pinot by linkedin.

the class AnomalyResource method viewMergedAnomaliesInRange.

// View merged anomalies for collection
@GET
@Path("/anomalies/view")
public List<MergedAnomalyResultDTO> viewMergedAnomaliesInRange(@NotNull @QueryParam("dataset") String dataset, @QueryParam("startTimeIso") String startTimeIso, @QueryParam("endTimeIso") String endTimeIso, @QueryParam("metric") String metric, @QueryParam("dimensions") String exploredDimensions, @DefaultValue("true") @QueryParam("applyAlertFilter") boolean applyAlertFiler) {
    if (StringUtils.isBlank(dataset)) {
        throw new IllegalArgumentException("dataset is a required query param");
    }
    DateTime endTime = DateTime.now();
    if (StringUtils.isNotEmpty(endTimeIso)) {
        endTime = ISODateTimeFormat.dateTimeParser().parseDateTime(endTimeIso);
    }
    DateTime startTime = endTime.minusDays(7);
    if (StringUtils.isNotEmpty(startTimeIso)) {
        startTime = ISODateTimeFormat.dateTimeParser().parseDateTime(startTimeIso);
    }
    List<MergedAnomalyResultDTO> anomalyResults = new ArrayList<>();
    try {
        if (StringUtils.isNotBlank(exploredDimensions)) {
            // Decode dimensions map from request, which may contain encode symbols such as "%20D", etc.
            exploredDimensions = URLDecoder.decode(exploredDimensions, UTF8);
            try {
                // Ensure the dimension names are sorted in order to match the string in backend database
                DimensionMap sortedDimensions = OBJECT_MAPPER.readValue(exploredDimensions, DimensionMap.class);
                exploredDimensions = OBJECT_MAPPER.writeValueAsString(sortedDimensions);
            } catch (IOException e) {
                LOG.warn("exploreDimensions may not be sorted because failed to read it as a json string: {}", e.toString());
            }
        }
        boolean loadRawAnomalies = false;
        if (StringUtils.isNotBlank(metric)) {
            if (StringUtils.isNotBlank(exploredDimensions)) {
                anomalyResults = anomalyMergedResultDAO.findByCollectionMetricDimensionsTime(dataset, metric, exploredDimensions, startTime.getMillis(), endTime.getMillis(), loadRawAnomalies);
            } else {
                anomalyResults = anomalyMergedResultDAO.findByCollectionMetricTime(dataset, metric, startTime.getMillis(), endTime.getMillis(), loadRawAnomalies);
            }
        } else {
            anomalyResults = anomalyMergedResultDAO.findByCollectionTime(dataset, startTime.getMillis(), endTime.getMillis(), loadRawAnomalies);
        }
    } catch (Exception e) {
        LOG.error("Exception in fetching anomalies", e);
    }
    if (applyAlertFiler) {
        // TODO: why need try catch?
        try {
            anomalyResults = AlertFilterHelper.applyFiltrationRule(anomalyResults, alertFilterFactory);
        } catch (Exception e) {
            LOG.warn("Failed to apply alert filters on anomalies for dataset:{}, metric:{}, start:{}, end:{}, exception:{}", dataset, metric, startTimeIso, endTimeIso, e);
        }
    }
    return anomalyResults;
}

Example 18 with DimensionMap

use of com.linkedin.thirdeye.api.DimensionMap in project pinot by linkedin.

the class DetectionTaskRunner method dimensionalShuffleAndUnifyAnalyze.

private ListMultimap<DimensionMap, RawAnomalyResultDTO> dimensionalShuffleAndUnifyAnalyze(DateTime windowStart, DateTime windowEnd, AnomalyDetectionInputContext anomalyDetectionInputContext) {
    int anomalyCounter = 0;
    ListMultimap<DimensionMap, RawAnomalyResultDTO> resultRawAnomalies = ArrayListMultimap.create();
    for (DimensionMap dimensionMap : anomalyDetectionInputContext.getDimensionKeyMetricTimeSeriesMap().keySet()) {
        List<RawAnomalyResultDTO> resultsOfAnEntry = runAnalyze(windowStart, windowEnd, anomalyDetectionInputContext, dimensionMap);
        // Set raw anomalies' properties
        handleResults(resultsOfAnEntry);
        LOG.info("Dimension {} has {} anomalies in window {} to {}", dimensionMap, resultsOfAnEntry.size(), windowStart, windowEnd);
        anomalyCounter += resultsOfAnEntry.size();
        resultRawAnomalies.putAll(dimensionMap, resultsOfAnEntry);
    }
    LOG.info("{} anomalies found in total", anomalyCounter);
    return resultRawAnomalies;
}

Example 19 with DimensionMap

use of com.linkedin.thirdeye.api.DimensionMap in project pinot by linkedin.

the class AnomalyMergeExecutor method performMergeBasedOnFunctionIdAndDimensions.

private void performMergeBasedOnFunctionIdAndDimensions(AnomalyFunctionDTO function, AnomalyMergeConfig mergeConfig, List<RawAnomalyResultDTO> unmergedResults, List<MergedAnomalyResultDTO> output) {
    Map<DimensionMap, List<RawAnomalyResultDTO>> dimensionsResultMap = new HashMap<>();
    for (RawAnomalyResultDTO anomalyResult : unmergedResults) {
        DimensionMap exploredDimensions = anomalyResult.getDimensions();
        if (!dimensionsResultMap.containsKey(exploredDimensions)) {
            dimensionsResultMap.put(exploredDimensions, new ArrayList<>());
        }
        dimensionsResultMap.get(exploredDimensions).add(anomalyResult);
    }
    for (DimensionMap exploredDimensions : dimensionsResultMap.keySet()) {
        List<RawAnomalyResultDTO> unmergedResultsByDimensions = dimensionsResultMap.get(exploredDimensions);
        long anomalyWindowStart = Long.MAX_VALUE;
        long anomalyWindowEnd = Long.MIN_VALUE;
        for (RawAnomalyResultDTO unmergedResultsByDimension : unmergedResultsByDimensions) {
            anomalyWindowStart = Math.min(anomalyWindowStart, unmergedResultsByDimension.getStartTime());
            anomalyWindowEnd = Math.max(anomalyWindowEnd, unmergedResultsByDimension.getEndTime());
        }
        // NOTE: We get "latest overlapped (Conflict)" merged anomaly instead of "latest" merged anomaly in order to
        // prevent the merge results of current (online) detection interfere the merge results of back-fill (offline)
        // detection.
        // Moreover, the window start is modified by mergeConfig.getSequentialAllowedGap() in order to allow a gap between
        // anomalies to be merged.
        MergedAnomalyResultDTO latestOverlappedMergedResult = mergedResultDAO.findLatestConflictByFunctionIdDimensions(function.getId(), exploredDimensions.toString(), anomalyWindowStart - mergeConfig.getSequentialAllowedGap(), anomalyWindowEnd);
        List<MergedAnomalyResultDTO> mergedResults = AnomalyTimeBasedSummarizer.mergeAnomalies(latestOverlappedMergedResult, unmergedResultsByDimensions, mergeConfig.getMaxMergeDurationLength(), mergeConfig.getSequentialAllowedGap());
        for (MergedAnomalyResultDTO mergedResult : mergedResults) {
            mergedResult.setFunction(function);
            mergedResult.setDimensions(exploredDimensions);
        }
        LOG.info("Merging [{}] raw anomalies into [{}] merged anomalies for function id : [{}] and dimensions : [{}]", unmergedResultsByDimensions.size(), mergedResults.size(), function.getId(), exploredDimensions);
        output.addAll(mergedResults);
    }
}

Example 20 with DimensionMap

use of com.linkedin.thirdeye.api.DimensionMap in project pinot by linkedin.

the class DetectionTaskRunner method fetchData.

private AnomalyDetectionInputContext fetchData(DateTime windowStart, DateTime windowEnd) throws JobExecutionException, ExecutionException {
    AnomalyDetectionInputContext adContext = new AnomalyDetectionInputContext();
    // Get Time Series
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis());
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    Map<DimensionMap, MetricTimeSeries> dimensionMapMetricTimeSeriesMap = new HashMap<>();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        // If the current time series belongs to OTHER dimension, which consists of time series whose
        // sum of all its values belows 1% of sum of all time series values, then its anomaly is
        // meaningless and hence we don't want to detection anomalies on it.
        String[] dimensionValues = dimensionKey.getDimensionValues();
        boolean isOTHERDimension = false;
        for (String dimensionValue : dimensionValues) {
            if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER) || dimensionValue.equalsIgnoreCase(ResponseParserUtils.UNKNOWN)) {
                isOTHERDimension = true;
                break;
            }
        }
        if (isOTHERDimension) {
            continue;
        }
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        dimensionMapMetricTimeSeriesMap.put(dimensionMap, entry.getValue());
        if (entry.getValue().getTimeWindowSet().size() < 1) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
    }
    adContext.setDimensionKeyMetricTimeSeriesMap(dimensionMapMetricTimeSeriesMap);
    // Get existing anomalies for this time range and this function id for all combinations of dimensions
    List<MergedAnomalyResultDTO> knownMergedAnomalies;
    if (anomalyFunction.useHistoryAnomaly()) {
        // if this anomaly function uses history data, then we get all time ranges
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    } else {
        // otherwise, we only get the merge anomaly for current window in order to remove duplicate raw anomalies
        List<Pair<Long, Long>> currentTimeRange = new ArrayList<>();
        currentTimeRange.add(new Pair<>(windowStart.getMillis(), windowEnd.getMillis()));
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), currentTimeRange);
    }
    // Sort the known merged and raw anomalies by their dimension names
    ArrayListMultimap<DimensionMap, MergedAnomalyResultDTO> dimensionMapToKnownMergedAnomalies = ArrayListMultimap.create();
    for (MergedAnomalyResultDTO knownMergedAnomaly : knownMergedAnomalies) {
        dimensionMapToKnownMergedAnomalies.put(knownMergedAnomaly.getDimensions(), knownMergedAnomaly);
    }
    adContext.setKnownMergedAnomalies(dimensionMapToKnownMergedAnomalies);
    // We always find existing raw anomalies to prevent duplicate raw anomalies are generated
    List<RawAnomalyResultDTO> existingRawAnomalies = getExistingRawAnomalies(anomalyFunctionSpec.getId(), windowStart.getMillis(), windowEnd.getMillis());
    ArrayListMultimap<DimensionMap, RawAnomalyResultDTO> dimensionNamesToKnownRawAnomalies = ArrayListMultimap.create();
    for (RawAnomalyResultDTO existingRawAnomaly : existingRawAnomalies) {
        dimensionNamesToKnownRawAnomalies.put(existingRawAnomaly.getDimensions(), existingRawAnomaly);
    }
    adContext.setExistingRawAnomalies(dimensionNamesToKnownRawAnomalies);
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(DAO_REGISTRY.getOverrideConfigDAO(), anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    adContext.setScalingFactors(scalingFactors);
    return adContext;
}