Examples with RawAnomalyResultDTO com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO used on opensource projects

Example 36 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class DetectionTaskRunner method fetchData.

private AnomalyDetectionInputContext fetchData(DateTime windowStart, DateTime windowEnd) throws JobExecutionException, ExecutionException {
    AnomalyDetectionInputContext adContext = new AnomalyDetectionInputContext();
    // Get Time Series
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis());
    Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesUtil.getTimeSeriesForAnomalyDetection(anomalyFunctionSpec, startEndTimeRanges);
    Map<DimensionMap, MetricTimeSeries> dimensionMapMetricTimeSeriesMap = new HashMap<>();
    for (Map.Entry<DimensionKey, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) {
        DimensionKey dimensionKey = entry.getKey();
        // If the current time series belongs to OTHER dimension, which consists of time series whose
        // sum of all its values belows 1% of sum of all time series values, then its anomaly is
        // meaningless and hence we don't want to detection anomalies on it.
        String[] dimensionValues = dimensionKey.getDimensionValues();
        boolean isOTHERDimension = false;
        for (String dimensionValue : dimensionValues) {
            if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER) || dimensionValue.equalsIgnoreCase(ResponseParserUtils.UNKNOWN)) {
                isOTHERDimension = true;
                break;
            }
        }
        if (isOTHERDimension) {
            continue;
        }
        DimensionMap dimensionMap = DimensionMap.fromDimensionKey(dimensionKey, collectionDimensions);
        dimensionMapMetricTimeSeriesMap.put(dimensionMap, entry.getValue());
        if (entry.getValue().getTimeWindowSet().size() < 1) {
            LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap);
            continue;
        }
    }
    adContext.setDimensionKeyMetricTimeSeriesMap(dimensionMapMetricTimeSeriesMap);
    // Get existing anomalies for this time range and this function id for all combinations of dimensions
    List<MergedAnomalyResultDTO> knownMergedAnomalies;
    if (anomalyFunction.useHistoryAnomaly()) {
        // if this anomaly function uses history data, then we get all time ranges
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    } else {
        // otherwise, we only get the merge anomaly for current window in order to remove duplicate raw anomalies
        List<Pair<Long, Long>> currentTimeRange = new ArrayList<>();
        currentTimeRange.add(new Pair<>(windowStart.getMillis(), windowEnd.getMillis()));
        knownMergedAnomalies = getKnownMergedAnomalies(anomalyFunctionSpec.getId(), currentTimeRange);
    }
    // Sort the known merged and raw anomalies by their dimension names
    ArrayListMultimap<DimensionMap, MergedAnomalyResultDTO> dimensionMapToKnownMergedAnomalies = ArrayListMultimap.create();
    for (MergedAnomalyResultDTO knownMergedAnomaly : knownMergedAnomalies) {
        dimensionMapToKnownMergedAnomalies.put(knownMergedAnomaly.getDimensions(), knownMergedAnomaly);
    }
    adContext.setKnownMergedAnomalies(dimensionMapToKnownMergedAnomalies);
    // We always find existing raw anomalies to prevent duplicate raw anomalies are generated
    List<RawAnomalyResultDTO> existingRawAnomalies = getExistingRawAnomalies(anomalyFunctionSpec.getId(), windowStart.getMillis(), windowEnd.getMillis());
    ArrayListMultimap<DimensionMap, RawAnomalyResultDTO> dimensionNamesToKnownRawAnomalies = ArrayListMultimap.create();
    for (RawAnomalyResultDTO existingRawAnomaly : existingRawAnomalies) {
        dimensionNamesToKnownRawAnomalies.put(existingRawAnomaly.getDimensions(), existingRawAnomaly);
    }
    adContext.setExistingRawAnomalies(dimensionNamesToKnownRawAnomalies);
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(DAO_REGISTRY.getOverrideConfigDAO(), anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
    adContext.setScalingFactors(scalingFactors);
    return adContext;
}

Example 37 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class AnomalyResource method deleteAnomalyFunctions.

// Delete anomaly function
@DELETE
@Path("/anomaly-function/delete")
public Response deleteAnomalyFunctions(@NotNull @QueryParam("id") Long id, @QueryParam("functionName") String functionName) throws IOException {
    if (id == null) {
        throw new IllegalArgumentException("id is a required query param");
    }
    // call endpoint to stop if active
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyFunctionDAO.findById(id);
    if (anomalyFunctionSpec == null) {
        throw new IllegalStateException("No anomalyFunctionSpec with id " + id);
    }
    // delete dependent entities
    // email config mapping
    List<EmailConfigurationDTO> emailConfigurations = emailConfigurationDAO.findByFunctionId(id);
    for (EmailConfigurationDTO emailConfiguration : emailConfigurations) {
        emailConfiguration.getFunctions().remove(anomalyFunctionSpec);
        emailConfigurationDAO.update(emailConfiguration);
    }
    // raw result mapping
    List<RawAnomalyResultDTO> rawResults = rawAnomalyResultDAO.findAllByTimeAndFunctionId(0, System.currentTimeMillis(), id);
    for (RawAnomalyResultDTO result : rawResults) {
        rawAnomalyResultDAO.delete(result);
    }
    // merged anomaly mapping
    List<MergedAnomalyResultDTO> mergedResults = anomalyMergedResultDAO.findByFunctionId(id);
    for (MergedAnomalyResultDTO result : mergedResults) {
        anomalyMergedResultDAO.delete(result);
    }
    // delete from db
    anomalyFunctionDAO.deleteById(id);
    return Response.noContent().build();
}

Example 38 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class OnboardResource method getExistingRawAnomalies.

/**
   * Show the content of merged anomalies whose start time is located in the given time ranges
   *
   * @param monitoringWindowStartTime The start time of the monitoring window (in milli-second)
   * @param monitoringWindowEndTime The start time of the monitoring window (in milli-second)
   */
@POST
@Path("function/{id}/getExistingRawAnomalies")
public List<RawAnomalyResultDTO> getExistingRawAnomalies(@PathParam("id") String id, @QueryParam("start") long monitoringWindowStartTime, @QueryParam("end") long monitoringWindowEndTime) {
    LOG.info("Retrieving merged anomaly results in the time range: {} -- {}", new DateTime(monitoringWindowStartTime), new DateTime(monitoringWindowEndTime));
    List<RawAnomalyResultDTO> rawResults = null;
    long functionId = Long.valueOf(id);
    AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
    if (anomalyFunction == null) {
        LOG.info("Anomaly functionId {} is not found", functionId);
        return rawResults;
    }
    rawResults = rawAnomalyResultDAO.findAllByTimeAndFunctionId(monitoringWindowStartTime, monitoringWindowEndTime, functionId);
    return rawResults;
}

Example 39 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class RawAnomalyResultManagerImpl method findByPredicate.

private List<RawAnomalyResultDTO> findByPredicate(Predicate predicate) {
    List<RawAnomalyResultBean> list = genericPojoDao.get(predicate, RawAnomalyResultBean.class);
    List<RawAnomalyResultDTO> result = new ArrayList<>();
    for (RawAnomalyResultBean bean : list) {
        result.add(createRawAnomalyDTOFromBean(bean));
    }
    return result;
}

Example 40 with RawAnomalyResultDTO

use of com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO in project pinot by linkedin.

the class TestMinMaxThresholdFunction method analyze.

@Test(dataProvider = "timeSeriesDataProvider")
public void analyze(Properties properties, TimeSeriesKey timeSeriesKey, long bucketSizeInMs, TimeSeries observedTimeSeries) throws Exception {
    AnomalyDetectionContext anomalyDetectionContext = new AnomalyDetectionContext();
    anomalyDetectionContext.setBucketSizeInMS(bucketSizeInMs);
    properties.put(MinMaxThresholdDetectionModel.MAX_VAL, "20");
    properties.put(MinMaxThresholdDetectionModel.MIN_VAL, "12");
    // Create anomaly function spec
    AnomalyFunctionDTO functionSpec = new AnomalyFunctionDTO();
    functionSpec.setMetric(mainMetric);
    functionSpec.setProperties(TestWeekOverWeekRuleFunction.toString(properties));
    AnomalyDetectionFunction function = new MinMaxThresholdFunction();
    function.init(functionSpec);
    anomalyDetectionContext.setAnomalyDetectionFunction(function);
    anomalyDetectionContext.setCurrent(mainMetric, observedTimeSeries);
    anomalyDetectionContext.setTimeSeriesKey(timeSeriesKey);
    List<RawAnomalyResultDTO> actualAnomalyResults = function.analyze(anomalyDetectionContext);
    // Expected RawAnomalies of WoW without smoothing
    List<RawAnomalyResultDTO> expectedRawAnomalies = new ArrayList<>();
    RawAnomalyResultDTO rawAnomaly1 = new RawAnomalyResultDTO();
    rawAnomaly1.setStartTime(observedStartTime);
    rawAnomaly1.setEndTime(observedStartTime + bucketMillis);
    rawAnomaly1.setWeight(-0.166666d);
    rawAnomaly1.setScore(13.6d);
    expectedRawAnomalies.add(rawAnomaly1);
    RawAnomalyResultDTO rawAnomaly2 = new RawAnomalyResultDTO();
    rawAnomaly2.setStartTime(observedStartTime + bucketMillis * 3);
    rawAnomaly2.setEndTime(observedStartTime + bucketMillis * 4);
    rawAnomaly2.setWeight(0.1d);
    rawAnomaly2.setScore(13.6d);
    expectedRawAnomalies.add(rawAnomaly2);
    RawAnomalyResultDTO rawAnomaly3 = new RawAnomalyResultDTO();
    rawAnomaly3.setStartTime(observedStartTime + bucketMillis * 4);
    rawAnomaly3.setEndTime(observedStartTime + bucketMillis * 5);
    rawAnomaly3.setWeight(-0.33333d);
    rawAnomaly3.setScore(13.6d);
    expectedRawAnomalies.add(rawAnomaly3);
    Assert.assertEquals(actualAnomalyResults.size(), expectedRawAnomalies.size());
    for (int i = 0; i < actualAnomalyResults.size(); ++i) {
        RawAnomalyResultDTO actualAnomaly = actualAnomalyResults.get(i);
        RawAnomalyResultDTO expectedAnomaly = actualAnomalyResults.get(i);
        Assert.assertEquals(actualAnomaly.getWeight(), expectedAnomaly.getWeight(), EPSILON);
        Assert.assertEquals(actualAnomaly.getScore(), expectedAnomaly.getScore(), EPSILON);
    }
    // Test getTimeSeriesIntervals
    List<Interval> expectedDataRanges = new ArrayList<>();
    expectedDataRanges.add(new Interval(observedStartTime, observedStartTime + bucketMillis * 5));
    List<Interval> actualDataRanges = function.getTimeSeriesIntervals(observedStartTime, observedStartTime + bucketMillis * 5);
    Assert.assertEquals(actualDataRanges, expectedDataRanges);
}