Examples with TimeGranularity com.linkedin.thirdeye.api.TimeGranularity used on opensource projects

Example 36 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class DetectionJobSchedulerUtils method getBucketSizeInMSForDataset.

/**
   * get bucket size in millis, according to data granularity of dataset
   * Bucket size are 1 HOUR for hourly, 1 DAY for daily
   * For MINUTE level data, bucket size is calculated based on anomaly function frequency
   * @param timeSpec
   * @return
   */
public static long getBucketSizeInMSForDataset(DatasetConfigDTO datasetConfig, AnomalyFunctionDTO anomalyFunction) {
    long bucketMillis = 0;
    TimeSpec timeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    TimeUnit dataUnit = timeSpec.getDataGranularity().getUnit();
    TimeGranularity functionFrequency = anomalyFunction.getFrequency();
    // Calculate time periods according to the function frequency
    if (dataUnit.equals(TimeUnit.MINUTES) || dataUnit.equals(TimeUnit.MILLISECONDS) || dataUnit.equals(TimeUnit.SECONDS)) {
        if (functionFrequency.getUnit().equals(TimeUnit.MINUTES) && (functionFrequency.getSize() <= 30)) {
            bucketMillis = TimeUnit.MILLISECONDS.convert(functionFrequency.getSize(), TimeUnit.MINUTES);
        } else {
            // default to HOURS
            bucketMillis = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
        }
    } else {
        bucketMillis = TimeUnit.MILLISECONDS.convert(1, dataUnit);
    }
    return bucketMillis;
}

Example 37 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class DetectionJobSchedulerUtils method getBoundaryAlignedTimeForDataset.

/**
   * round this time to earlier boundary, depending on granularity of dataset
   * e.g. 12:15pm on HOURLY dataset should be treated as 12pm
   * any dataset with granularity finer than HOUR, will be rounded as per function frequency (assumption is that this is in MINUTES)
   * so 12.53 on 5 MINUTES dataset, with function frequency 15 MINUTES will be rounded to 12.45
   * @param anomalyFunction
   * @param timeSpec
   * @param dataCompletenessStartTime
   * @param dateTimeZone
   * @return
   */
public static long getBoundaryAlignedTimeForDataset(DatasetConfigDTO datasetConfig, DateTime dateTime, AnomalyFunctionDTO anomalyFunction) {
    TimeSpec timeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    TimeUnit dataUnit = timeSpec.getDataGranularity().getUnit();
    TimeGranularity functionFrequency = anomalyFunction.getFrequency();
    // Calculate time periods according to the function frequency
    if (dataUnit.equals(TimeUnit.MINUTES) || dataUnit.equals(TimeUnit.MILLISECONDS) || dataUnit.equals(TimeUnit.SECONDS)) {
        if (functionFrequency.getUnit().equals(TimeUnit.MINUTES) && (functionFrequency.getSize() <= 30)) {
            int minuteBucketSize = functionFrequency.getSize();
            int roundedMinutes = (dateTime.getMinuteOfHour() / minuteBucketSize) * minuteBucketSize;
            dateTime = dateTime.withTime(dateTime.getHourOfDay(), roundedMinutes, 0, 0);
        } else {
            // default to HOURS
            dateTime = getBoundaryAlignedTimeForDataset(dateTime, TimeUnit.HOURS);
        }
    } else {
        dateTime = getBoundaryAlignedTimeForDataset(dateTime, dataUnit);
    }
    return dateTime.getMillis();
}

Example 38 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class TimeSeriesUtil method getTimeSeriesForAnomalyDetection.

/**
   * Returns the set of metric time series that are needed by the given anomaly function for detecting anomalies.
   *
   * The time granularity is the granularity of the function's collection, i.e., the buckets are not aggregated,
   * in order to increase the accuracy for detecting anomalies.
   *
   * @param anomalyFunctionSpec spec of the anomaly function
   * @param startEndTimeRanges the time ranges to retrieve the data for constructing the time series
   *
   * @return the data that is needed by the anomaly function for detecting anomalies.
   * @throws JobExecutionException
   * @throws ExecutionException
   */
public static Map<DimensionKey, MetricTimeSeries> getTimeSeriesForAnomalyDetection(AnomalyFunctionDTO anomalyFunctionSpec, List<Pair<Long, Long>> startEndTimeRanges) throws JobExecutionException, ExecutionException {
    String filterString = anomalyFunctionSpec.getFilters();
    Multimap<String, String> filters;
    if (StringUtils.isNotBlank(filterString)) {
        filters = ThirdEyeUtils.getFilterSet(filterString);
    } else {
        filters = HashMultimap.create();
    }
    List<String> groupByDimensions;
    String exploreDimensionString = anomalyFunctionSpec.getExploreDimensions();
    if (StringUtils.isNotBlank(exploreDimensionString)) {
        groupByDimensions = Arrays.asList(exploreDimensionString.trim().split(","));
    } else {
        groupByDimensions = Collections.emptyList();
    }
    TimeGranularity timeGranularity = new TimeGranularity(anomalyFunctionSpec.getBucketSize(), anomalyFunctionSpec.getBucketUnit());
    TimeSeriesResponse timeSeriesResponse = getTimeSeriesResponseImpl(anomalyFunctionSpec, startEndTimeRanges, timeGranularity, filters, groupByDimensions, false);
    try {
        Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesResponseConverter.toMap(timeSeriesResponse, Utils.getSchemaDimensionNames(anomalyFunctionSpec.getCollection()));
        return dimensionKeyMetricTimeSeriesMap;
    } catch (Exception e) {
        LOG.info("Failed to get schema dimensions for constructing dimension keys:", e.toString());
        return Collections.emptyMap();
    }
}