Examples with TimeGranularity com.linkedin.thirdeye.api.TimeGranularity used on opensource projects

Example 1 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class AnomalyMergeExecutor method updateMergedAnomalyWeight.

/**
   * Uses function-specific method to re-computes the weight of merged anomaly.
   *
   * @param anomalyMergedResult the merged anomaly to be updated
   * @param mergeConfig the merge configuration that was applied when merge the merged anomaly
   * @throws Exception if error occurs when retrieving the time series for calculating the weight
   */
private void updateMergedAnomalyWeight(MergedAnomalyResultDTO anomalyMergedResult, AnomalyMergeConfig mergeConfig) throws Exception {
    AnomalyFunctionDTO anomalyFunctionSpec = anomalyMergedResult.getFunction();
    BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec);
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(anomalyMergedResult.getStartTime(), anomalyMergedResult.getEndTime());
    TimeGranularity timeGranularity = new TimeGranularity(anomalyFunctionSpec.getBucketSize(), anomalyFunctionSpec.getBucketUnit());
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(anomalyFunctionSpec, startEndTimeRanges, anomalyMergedResult.getDimensions(), timeGranularity, false);
    if (metricTimeSeries != null) {
        DateTime windowStart = new DateTime(anomalyMergedResult.getStartTime());
        DateTime windowEnd = new DateTime(anomalyMergedResult.getEndTime());
        List<MergedAnomalyResultDTO> knownAnomalies = Collections.emptyList();
        // Retrieve history merged anomalies
        if (anomalyFunction.useHistoryAnomaly()) {
            switch(mergeConfig.getMergeStrategy()) {
                case FUNCTION:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis());
                    break;
                case FUNCTION_DIMENSIONS:
                    knownAnomalies = getHistoryMergedAnomalies(anomalyFunction, windowStart.getMillis(), windowEnd.getMillis(), anomalyMergedResult.getDimensions());
                    break;
                default:
                    throw new IllegalArgumentException("Merge strategy " + mergeConfig.getMergeStrategy() + " not supported");
            }
            if (knownAnomalies.size() > 0) {
                LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
                LOG.info("Checking if any known anomalies overlap with the monitoring window of anomaly detection, which could result in unwanted holes in current values.");
                AnomalyUtils.logAnomaliesOverlapWithWindow(windowStart, windowEnd, knownAnomalies);
            }
        }
        // Transform Time Series
        List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, anomalyFunctionSpec.getCollection(), anomalyFunctionSpec.getTopicMetric(), anomalyFunctionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStart.getMillis(), windowEnd.getMillis()));
        if (CollectionUtils.isNotEmpty(scalingFactors)) {
            Properties properties = anomalyFunction.getProperties();
            MetricTransfer.rescaleMetric(metricTimeSeries, windowStart.getMillis(), scalingFactors, anomalyFunctionSpec.getTopicMetric(), properties);
        }
        anomalyFunction.updateMergedAnomalyInfo(anomalyMergedResult, metricTimeSeries, windowStart, windowEnd, knownAnomalies);
    }
}

Example 2 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class TimeBasedAnomalyMerger method fetchDataByDimension.

/**
   * Fetch time series, known merged anomalies, and scaling factor for the specified dimension. Note that scaling
   * factor has no dimension information, so all scaling factor in the specified time range will be retrieved.
   *
   * @param windowStartTime the start time for retrieving the data
   * @param windowEndTime the end time for retrieving the data
   * @param dimensions the dimension of the data
   * @param anomalyFunction the anomaly function that produces the anomaly
   * @param mergedResultDAO DAO for merged anomalies
   * @param overrideConfigDAO DAO for override configuration
   * @param endTimeInclusive set to true if the end time should be inclusive; mainly used by the queries from UI
   * @return an anomaly detection input context that contains all the retrieved data
   * @throws Exception if it fails to retrieve time series from DB.
   */
public static AnomalyDetectionInputContext fetchDataByDimension(long windowStartTime, long windowEndTime, DimensionMap dimensions, BaseAnomalyFunction anomalyFunction, MergedAnomalyResultManager mergedResultDAO, OverrideConfigManager overrideConfigDAO, boolean endTimeInclusive) throws Exception {
    AnomalyFunctionDTO functionSpec = anomalyFunction.getSpec();
    List<Pair<Long, Long>> startEndTimeRanges = anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime);
    TimeGranularity timeGranularity = new TimeGranularity(functionSpec.getBucketSize(), functionSpec.getBucketUnit());
    AnomalyDetectionInputContext adInputContext = new AnomalyDetectionInputContext();
    // Retrieve Time Series
    MetricTimeSeries metricTimeSeries = TimeSeriesUtil.getTimeSeriesByDimension(functionSpec, startEndTimeRanges, dimensions, timeGranularity, endTimeInclusive);
    Map<DimensionMap, MetricTimeSeries> metricTimeSeriesMap = new HashMap<>();
    metricTimeSeriesMap.put(dimensions, metricTimeSeries);
    adInputContext.setDimensionKeyMetricTimeSeriesMap(metricTimeSeriesMap);
    // Retrieve historical anomaly
    if (anomalyFunction.useHistoryAnomaly()) {
        List<MergedAnomalyResultDTO> knownAnomalies = getBaselineKnownAnomaliesByDimension(anomalyFunction, windowStartTime, windowEndTime, dimensions, mergedResultDAO);
        ListMultimap<DimensionMap, MergedAnomalyResultDTO> mergedAnomalyMap = ArrayListMultimap.create();
        mergedAnomalyMap.putAll(dimensions, knownAnomalies);
        adInputContext.setKnownMergedAnomalies(mergedAnomalyMap);
        if (knownAnomalies.size() > 0) {
            LOG.info("Found {} history anomalies for computing the weight of current merged anomaly.", knownAnomalies.size());
        }
    }
    // Retrieve scaling factor
    List<ScalingFactor> scalingFactors = OverrideConfigHelper.getTimeSeriesScalingFactors(overrideConfigDAO, functionSpec.getCollection(), functionSpec.getTopicMetric(), functionSpec.getId(), anomalyFunction.getDataRangeIntervals(windowStartTime, windowEndTime));
    adInputContext.setScalingFactors(scalingFactors);
    return adInputContext;
}

Example 3 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class DetectionJobRunner method alignTimestampsToDataTimezone.

private DateTime alignTimestampsToDataTimezone(DateTime inputDateTime, String collection) {
    try {
        DatasetConfigDTO datasetConfig = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(collection);
        TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
        TimeGranularity dataGranularity = timespec.getDataGranularity();
        String timeFormat = timespec.getFormat();
        if (dataGranularity.getUnit().equals(TimeUnit.DAYS)) {
            DateTimeZone dataTimeZone = Utils.getDataTimeZone(collection);
            DateTimeFormatter inputDataDateTimeFormatter = DateTimeFormat.forPattern(timeFormat).withZone(dataTimeZone);
            long inputMillis = inputDateTime.getMillis();
            String inputDateTimeString = inputDataDateTimeFormatter.print(inputMillis);
            long timeZoneOffsetMillis = inputDataDateTimeFormatter.parseMillis(inputDateTimeString);
            inputDateTime = new DateTime(timeZoneOffsetMillis);
        }
    } catch (Exception e) {
        LOG.error("Exception in aligning timestamp to data time zone", e);
    }
    return inputDateTime;
}

Example 4 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class PinotThirdEyeClient method parseResultSets.

private List<String[]> parseResultSets(ThirdEyeRequest request, List<ResultSet> resultSets, List<MetricFunction> metricFunctions, List<String> dimensionNames, DatasetConfigDTO datasetConfig) throws ExecutionException {
    int numGroupByKeys = 0;
    boolean hasGroupBy = false;
    if (request.getGroupByTimeGranularity() != null) {
        numGroupByKeys += 1;
    }
    if (request.getGroupBy() != null) {
        numGroupByKeys += request.getGroupBy().size();
    }
    if (numGroupByKeys > 0) {
        hasGroupBy = true;
    }
    int numMetrics = request.getMetricFunctions().size();
    int numCols = numGroupByKeys + numMetrics;
    boolean hasGroupByTime = false;
    String collection = datasetConfig.getDataset();
    TimeGranularity dataGranularity = null;
    long startTime = request.getStartTimeInclusive().getMillis();
    DateTimeZone dateTimeZone = Utils.getDataTimeZone(collection);
    DateTime startDateTime = new DateTime(startTime, dateTimeZone);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    dataGranularity = timespec.getDataGranularity();
    boolean isISOFormat = false;
    DateTimeFormatter inputDataDateTimeFormatter = null;
    String timeFormat = timespec.getFormat();
    if (timeFormat != null && !timeFormat.equals(TimeSpec.SINCE_EPOCH_FORMAT)) {
        isISOFormat = true;
        inputDataDateTimeFormatter = DateTimeFormat.forPattern(timeFormat).withZone(dateTimeZone);
    }
    if (request.getGroupByTimeGranularity() != null) {
        hasGroupByTime = true;
    }
    LinkedHashMap<String, String[]> dataMap = new LinkedHashMap<>();
    for (int i = 0; i < resultSets.size(); i++) {
        ResultSet resultSet = resultSets.get(i);
        int numRows = resultSet.getRowCount();
        for (int r = 0; r < numRows; r++) {
            boolean skipRowDueToError = false;
            String[] groupKeys;
            if (hasGroupBy) {
                groupKeys = new String[resultSet.getGroupKeyLength()];
                for (int grpKeyIdx = 0; grpKeyIdx < resultSet.getGroupKeyLength(); grpKeyIdx++) {
                    String groupKeyVal = "";
                    try {
                        groupKeyVal = resultSet.getGroupKeyString(r, grpKeyIdx);
                    } catch (Exception e) {
                    // IGNORE FOR NOW, workaround for Pinot Bug
                    }
                    if (hasGroupByTime && grpKeyIdx == 0) {
                        int timeBucket;
                        long millis;
                        if (!isISOFormat) {
                            millis = dataGranularity.toMillis(Double.valueOf(groupKeyVal).longValue());
                        } else {
                            millis = DateTime.parse(groupKeyVal, inputDataDateTimeFormatter).getMillis();
                        }
                        if (millis < startTime) {
                            LOG.error("Data point earlier than requested start time {}: {}", new Date(startTime), new Date(millis));
                            skipRowDueToError = true;
                            break;
                        }
                        timeBucket = TimeRangeUtils.computeBucketIndex(request.getGroupByTimeGranularity(), startDateTime, new DateTime(millis, dateTimeZone));
                        groupKeyVal = String.valueOf(timeBucket);
                    }
                    groupKeys[grpKeyIdx] = groupKeyVal;
                }
                if (skipRowDueToError) {
                    continue;
                }
            } else {
                groupKeys = new String[] {};
            }
            StringBuilder groupKeyBuilder = new StringBuilder("");
            for (String grpKey : groupKeys) {
                groupKeyBuilder.append(grpKey).append("|");
            }
            String compositeGroupKey = groupKeyBuilder.toString();
            String[] rowValues = dataMap.get(compositeGroupKey);
            if (rowValues == null) {
                rowValues = new String[numCols];
                Arrays.fill(rowValues, "0");
                System.arraycopy(groupKeys, 0, rowValues, 0, groupKeys.length);
                dataMap.put(compositeGroupKey, rowValues);
            }
            rowValues[groupKeys.length + i] = String.valueOf(Double.parseDouble(rowValues[groupKeys.length + i]) + Double.parseDouble(resultSet.getString(r, 0)));
        }
    }
    List<String[]> rows = new ArrayList<>();
    rows.addAll(dataMap.values());
    return rows;
}

Example 5 with TimeGranularity

use of com.linkedin.thirdeye.api.TimeGranularity in project pinot by linkedin.

the class TimeOnTimeComparisonHandler method handle.

public TimeOnTimeComparisonResponse handle(TimeOnTimeComparisonRequest comparisonRequest) throws Exception {
    ThirdEyeRequestBuilder builder = new ThirdEyeRequestBuilder();
    builder.setCollection(comparisonRequest.getCollectionName());
    List<Range<DateTime>> baselineTimeranges = new ArrayList<>();
    List<Range<DateTime>> currentTimeranges = new ArrayList<>();
    TimeGranularity aggregationTimeGranularity = comparisonRequest.getAggregationTimeGranularity();
    // baseline time ranges
    DateTime baselineStart = comparisonRequest.getBaselineStart();
    DateTime baselineEnd = comparisonRequest.getBaselineEnd();
    // current time ranges
    DateTime currentStart = comparisonRequest.getCurrentStart();
    DateTime currentEnd = comparisonRequest.getCurrentEnd();
    if (comparisonRequest.isEndDateInclusive()) {
        // ThirdEyeRequest is exclusive endpoint, so increment end by one bucket
        currentEnd = TimeRangeUtils.increment(currentEnd, aggregationTimeGranularity);
        baselineEnd = TimeRangeUtils.increment(baselineEnd, aggregationTimeGranularity);
    }
    baselineTimeranges = TimeRangeUtils.computeTimeRanges(aggregationTimeGranularity, baselineStart, baselineEnd);
    currentTimeranges = TimeRangeUtils.computeTimeRanges(aggregationTimeGranularity, currentStart, currentEnd);
    // create baseline request
    ThirdEyeRequest baselineRequest = createThirdEyeRequest(BASELINE, comparisonRequest, baselineStart, baselineEnd);
    // create current request
    ThirdEyeRequest currentRequest = createThirdEyeRequest(CURRENT, comparisonRequest, currentStart, currentEnd);
    List<ThirdEyeRequest> requests = new ArrayList<>();
    requests.add(baselineRequest);
    requests.add(currentRequest);
    Map<ThirdEyeRequest, Future<ThirdEyeResponse>> futureResponseMap;
    futureResponseMap = queryCache.getQueryResultsAsync(requests);
    ThirdEyeResponse baselineResponse = null;
    ThirdEyeResponse currentResponse = null;
    for (Entry<ThirdEyeRequest, Future<ThirdEyeResponse>> entry : futureResponseMap.entrySet()) {
        ThirdEyeRequest request = entry.getKey();
        Future<ThirdEyeResponse> responseFuture = entry.getValue();
        ThirdEyeResponse response = responseFuture.get(60000, TimeUnit.SECONDS);
        if (BASELINE.equals(request.getRequestReference())) {
            baselineResponse = response;
        } else if (CURRENT.equals(request.getRequestReference())) {
            currentResponse = response;
        }
    }
    TimeOnTimeResponseParser timeOnTimeResponseParser = new TimeOnTimeResponseParser(baselineResponse, currentResponse, baselineTimeranges, currentTimeranges, comparisonRequest.getAggregationTimeGranularity(), comparisonRequest.getGroupByDimensions());
    List<Row> rows = timeOnTimeResponseParser.parseResponse();
    // compute the derived metrics
    computeDerivedMetrics(comparisonRequest, rows);
    return new TimeOnTimeComparisonResponse(rows);
}