Examples with AnomalyFunctionDTO com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO used on opensource projects

Example 66 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class TestDetectionJobSchedulerUtils method testGetNewEntriesForDetectionSchedulerMinuteLevel.

@Test
public void testGetNewEntriesForDetectionSchedulerMinuteLevel() throws Exception {
    DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
    datasetConfig.setTimeColumn("Date");
    datasetConfig.setTimeUnit(TimeUnit.MINUTES);
    datasetConfig.setTimeDuration(5);
    DateTimeZone dateTimeZone = DateTimeZone.UTC;
    AnomalyFunctionDTO anomalyFunction = new AnomalyFunctionDTO();
    anomalyFunction.setFrequency(new TimeGranularity(15, TimeUnit.MINUTES));
    DateTimeFormatter dateTimeFormatter = DetectionJobSchedulerUtils.getDateTimeFormatterForDataset(datasetConfig, dateTimeZone);
    String currentDateTimeString = "201702140336";
    String currentDateTimeStringRounded = "201702140330";
    DateTime currentDateTime = minuteDateTimeFormatter.parseDateTime(currentDateTimeString);
    DateTime currentDateTimeRounded = dateTimeFormatter.parseDateTime(currentDateTimeStringRounded);
    DetectionStatusDTO lastEntryForFunction = null;
    // null last entry
    Map<String, Long> newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry same as current time
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(currentDateTimeStringRounded);
    lastEntryForFunction.setDateToCheckInMS(currentDateTimeRounded.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 0);
    // last entry 15 MINUTES before current time
    String lastEntryDateTimeString = "201702140315";
    DateTime lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 1);
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
    // last entry 45 MINUTES  before current time
    lastEntryDateTimeString = "201702140245";
    lastEntryDateTime = dateTimeFormatter.parseDateTime(lastEntryDateTimeString);
    lastEntryForFunction = new DetectionStatusDTO();
    lastEntryForFunction.setDateToCheckInSDF(lastEntryDateTimeString);
    lastEntryForFunction.setDateToCheckInMS(lastEntryDateTime.getMillis());
    newEntries = DetectionJobSchedulerUtils.getNewEntries(currentDateTime, lastEntryForFunction, anomalyFunction, datasetConfig, dateTimeZone);
    Assert.assertEquals(newEntries.size(), 3);
    Assert.assertNotNull(newEntries.get("201702140300"));
    Assert.assertNotNull(newEntries.get("201702140315"));
    Assert.assertNotNull(newEntries.get("201702140330"));
    Assert.assertEquals(newEntries.get(currentDateTimeStringRounded), new Long(currentDateTimeRounded.getMillis()));
}

Example 67 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class AnomaliesResource method getAnomalyDetails.

/**
   * Generates Anomaly Details for each merged anomaly
   * @param mergedAnomaly
   * @param datasetConfig
   * @param timeSeriesDateFormatter
   * @param startEndDateFormatterHours
   * @param startEndDateFormatterDays
   * @param externalUrl
   * @return
   */
private AnomalyDetails getAnomalyDetails(MergedAnomalyResultDTO mergedAnomaly, DatasetConfigDTO datasetConfig, DateTimeFormatter timeSeriesDateFormatter, DateTimeFormatter startEndDateFormatterHours, DateTimeFormatter startEndDateFormatterDays, String externalUrl) throws Exception {
    String dataset = datasetConfig.getDataset();
    String metricName = mergedAnomaly.getMetric();
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyFunctionDAO.findById(mergedAnomaly.getFunctionId());
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    String aggGranularity = constructAggGranularity(datasetConfig);
    long anomalyStartTime = mergedAnomaly.getStartTime();
    long anomalyEndTime = mergedAnomaly.getEndTime();
    TimeRange range = getTimeseriesOffsetedTimes(anomalyStartTime, anomalyEndTime, datasetConfig);
    long currentStartTime = range.getStart();
    long currentEndTime = range.getEnd();
    DimensionMap dimensions = mergedAnomaly.getDimensions();
    TimeGranularity timeGranularity = Utils.getAggregationTimeGranularity(aggGranularity, anomalyFunctionSpec.getCollection());
    long bucketMillis = timeGranularity.toMillis();
    AnomalyDetails anomalyDetails = null;
    try {
        AnomalyDetectionInputContext adInputContext = TimeBasedAnomalyMerger.fetchDataByDimension(currentStartTime, currentEndTime, dimensions, anomalyFunction, mergedAnomalyResultDAO, overrideConfigDAO, true);
        MetricTimeSeries metricTimeSeries = adInputContext.getDimensionKeyMetricTimeSeriesMap().get(dimensions);
        // Transform time series with scaling factor
        List<ScalingFactor> scalingFactors = adInputContext.getScalingFactors();
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, currentStartTime, scalingFactors, anomalyFunctionSpec.getTopicMetric(), properties);
        }
        List<MergedAnomalyResultDTO> knownAnomalies = adInputContext.getKnownMergedAnomalies().get(dimensions);
        // Known anomalies are ignored (the null parameter) because 1. we can reduce users' waiting time and 2. presentation
        // data does not need to be as accurate as the one used for detecting anomalies
        AnomalyTimelinesView anomalyTimelinesView = anomalyFunction.getTimeSeriesView(metricTimeSeries, bucketMillis, anomalyFunctionSpec.getTopicMetric(), currentStartTime, currentEndTime, knownAnomalies);
        anomalyDetails = constructAnomalyDetails(metricName, dataset, datasetConfig, mergedAnomaly, anomalyFunctionSpec, currentStartTime, currentEndTime, anomalyTimelinesView, timeSeriesDateFormatter, startEndDateFormatterHours, startEndDateFormatterDays, externalUrl);
    } catch (Exception e) {
        LOG.error("Exception in constructing anomaly wrapper for anomaly {}", mergedAnomaly.getId(), e);
    }
    return anomalyDetails;
}

Example 68 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class AbstractManagerImpl method createRawAnomalyDTOFromBean.

protected RawAnomalyResultDTO createRawAnomalyDTOFromBean(RawAnomalyResultBean rawAnomalyResultBean) {
    RawAnomalyResultDTO rawAnomalyResultDTO;
    rawAnomalyResultDTO = MODEL_MAPPER.map(rawAnomalyResultBean, RawAnomalyResultDTO.class);
    if (rawAnomalyResultBean.getFunctionId() != null) {
        AnomalyFunctionBean anomalyFunctionBean = genericPojoDao.get(rawAnomalyResultBean.getFunctionId(), AnomalyFunctionBean.class);
        if (anomalyFunctionBean == null) {
            LOG.error("this anomaly function bean should not be null");
        }
        AnomalyFunctionDTO anomalyFunctionDTO = MODEL_MAPPER.map(anomalyFunctionBean, AnomalyFunctionDTO.class);
        rawAnomalyResultDTO.setFunction(anomalyFunctionDTO);
    }
    if (rawAnomalyResultBean.getAnomalyFeedbackId() != null) {
        AnomalyFeedbackBean anomalyFeedbackBean = genericPojoDao.get(rawAnomalyResultBean.getAnomalyFeedbackId(), AnomalyFeedbackBean.class);
        AnomalyFeedbackDTO anomalyFeedbackDTO = MODEL_MAPPER.map(anomalyFeedbackBean, AnomalyFeedbackDTO.class);
        rawAnomalyResultDTO.setFeedback(anomalyFeedbackDTO);
    }
    return rawAnomalyResultDTO;
}

Example 69 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class AnomalyFunctionManagerImpl method findAllActiveFunctions.

/*
   * (non-Javadoc)
   *
   * @see com.linkedin.thirdeye.datalayer.bao.IAnomalyFunctionManager# findAllActiveFunctions()
   */
@Override
public List<AnomalyFunctionDTO> findAllActiveFunctions() {
    Predicate predicate = Predicate.EQ("active", true);
    List<AnomalyFunctionBean> list = genericPojoDao.get(predicate, AnomalyFunctionBean.class);
    List<AnomalyFunctionDTO> result = new ArrayList<>();
    for (AnomalyFunctionBean abstractBean : list) {
        AnomalyFunctionDTO dto = MODEL_MAPPER.map(abstractBean, AnomalyFunctionDTO.class);
        result.add(dto);
    }
    return result;
}

Example 70 with AnomalyFunctionDTO

use of com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO in project pinot by linkedin.

the class AnomalyFunctionManagerImpl method findAllByCollection.

@Override
public List<AnomalyFunctionDTO> findAllByCollection(String collection) {
    Predicate predicate = Predicate.EQ("collection", collection);
    List<AnomalyFunctionBean> list = genericPojoDao.get(predicate, AnomalyFunctionBean.class);
    List<AnomalyFunctionDTO> result = new ArrayList<>();
    for (AnomalyFunctionBean abstractBean : list) {
        AnomalyFunctionDTO dto = MODEL_MAPPER.map(abstractBean, AnomalyFunctionDTO.class);
        result.add(dto);
    }
    return result;
}