Examples with TimeSeriesResponse com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse used on opensource projects

Example 1 with TimeSeriesResponse

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse in project pinot by linkedin.

the class TimeSeriesUtil method getTimeSeriesByDimension.

/**
   * Returns the metric time series that were given to the anomaly function for anomaly detection. If the dimension to
   * retrieve is OTHER, this method retrieves all combinations of dimensions and calculate the metric time series for
   * OTHER dimension on-the-fly.
   *
   * @param anomalyFunctionSpec spec of the anomaly function
   * @param startEndTimeRanges the time ranges to retrieve the data for constructing the time series
   * @param dimensionMap a dimension map that is used to construct the filter for retrieving the corresponding data
   *                     that was used to detected the anomaly
   * @param timeGranularity time granularity of the time series
   * @param endTimeInclusive set to true if the end time should be inclusive; mainly used by the query for UI
   * @return the time series in the same format as those used by the given anomaly function for anomaly detection
   *
   * @throws JobExecutionException
   * @throws ExecutionException
   */
public static MetricTimeSeries getTimeSeriesByDimension(AnomalyFunctionDTO anomalyFunctionSpec, List<Pair<Long, Long>> startEndTimeRanges, DimensionMap dimensionMap, TimeGranularity timeGranularity, boolean endTimeInclusive) throws JobExecutionException, ExecutionException {
    // Get the original filter
    Multimap<String, String> filters;
    String filterString = anomalyFunctionSpec.getFilters();
    if (StringUtils.isNotBlank(filterString)) {
        filters = ThirdEyeUtils.getFilterSet(filterString);
    } else {
        filters = HashMultimap.create();
    }
    // Decorate filters according to dimensionMap
    filters = ThirdEyeUtils.getFilterSetFromDimensionMap(dimensionMap, filters);
    boolean hasOTHERDimensionName = false;
    for (String dimensionValue : dimensionMap.values()) {
        if (dimensionValue.equalsIgnoreCase(ResponseParserUtils.OTHER)) {
            hasOTHERDimensionName = true;
            break;
        }
    }
    // groupByDimensions (i.e., exploreDimensions) is empty by default because the query for getting the time series
    // will have the decorated filters according to anomalies' explore dimensions.
    // However, if there exists any dimension with value "OTHER, then we need to honor the origin groupBy in order to
    // construct the data for OTHER
    List<String> groupByDimensions = Collections.emptyList();
    if (hasOTHERDimensionName && StringUtils.isNotBlank(anomalyFunctionSpec.getExploreDimensions().trim())) {
        groupByDimensions = Arrays.asList(anomalyFunctionSpec.getExploreDimensions().trim().split(","));
    }
    TimeSeriesResponse response = getTimeSeriesResponseImpl(anomalyFunctionSpec, startEndTimeRanges, timeGranularity, filters, groupByDimensions, endTimeInclusive);
    try {
        Map<DimensionKey, MetricTimeSeries> metricTimeSeriesMap = TimeSeriesResponseConverter.toMap(response, Utils.getSchemaDimensionNames(anomalyFunctionSpec.getCollection()));
        return extractMetricTimeSeriesByDimension(metricTimeSeriesMap);
    } catch (Exception e) {
        LOG.warn("Unable to get schema dimension name for retrieving metric time series: {}", e.toString());
        return null;
    }
}

Example 2 with TimeSeriesResponse

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse in project pinot by linkedin.

the class AnomaliesResource method getTimeSeriesData.

/**
   * Get timeseries for metric
   * @param collection
   * @param filters
   * @param start
   * @param end
   * @param aggTimeGranularity
   * @param metric
   * @return
   * @throws Exception
   */
private JSONObject getTimeSeriesData(String collection, Multimap<String, String> filters, Long start, Long end, String aggTimeGranularity, String metric) throws Exception {
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(collection);
    DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(collection);
    request.setStart(new DateTime(start, timeZoneForCollection));
    request.setEnd(new DateTime(end, timeZoneForCollection));
    request.setFilterSet(filters);
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metric, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getAggregationTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        request.setEndDateInclusive(true);
    }
    TimeSeriesHandler handler = new TimeSeriesHandler(CACHE_REGISTRY.getQueryCache());
    JSONObject jsonResponseObject = new JSONObject();
    try {
        TimeSeriesResponse response = handler.handle(request);
        JSONObject timeseriesMap = new JSONObject();
        JSONArray timeValueArray = new JSONArray();
        TreeSet<String> keys = new TreeSet<>();
        TreeSet<Long> times = new TreeSet<>();
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            times.add(timeSeriesRow.getStart());
        }
        for (Long time : times) {
            timeValueArray.put(time);
        }
        timeseriesMap.put("time", timeValueArray);
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            for (TimeSeriesMetric metricTimeSeries : timeSeriesRow.getMetrics()) {
                String key = metricTimeSeries.getMetricName();
                JSONArray valueArray;
                if (!timeseriesMap.has(key)) {
                    valueArray = new JSONArray();
                    timeseriesMap.put(key, valueArray);
                    keys.add(key);
                } else {
                    valueArray = timeseriesMap.getJSONArray(key);
                }
                valueArray.put(metricTimeSeries.getValue());
            }
        }
        JSONObject summaryMap = new JSONObject();
        summaryMap.put("currentStart", start);
        summaryMap.put("currentEnd", end);
        jsonResponseObject.put("timeSeriesData", timeseriesMap);
        jsonResponseObject.put("keys", new JSONArray(keys));
        jsonResponseObject.put("summary", summaryMap);
    } catch (Exception e) {
        throw e;
    }
    LOG.info("Response:{}", jsonResponseObject);
    return jsonResponseObject;
}

Example 3 with TimeSeriesResponse

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse in project pinot by linkedin.

the class TimeSeriesUtil method getTimeSeriesResponseImpl.

private static TimeSeriesResponse getTimeSeriesResponseImpl(AnomalyFunctionDTO anomalyFunctionSpec, List<Pair<Long, Long>> startEndTimeRanges, TimeGranularity timeGranularity, Multimap<String, String> filters, List<String> groupByDimensions, boolean endTimeInclusive) throws JobExecutionException, ExecutionException {
    TimeSeriesHandler timeSeriesHandler = new TimeSeriesHandler(ThirdEyeCacheRegistry.getInstance().getQueryCache());
    // Seed request with top-level...
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(anomalyFunctionSpec.getCollection());
    // TODO: Check low level support for multiple metrics retrieval
    String metricsToRetrieve = String.join(",", anomalyFunctionSpec.getMetrics());
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metricsToRetrieve, anomalyFunctionSpec.getMetricFunction(), anomalyFunctionSpec.getCollection());
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(timeGranularity);
    request.setEndDateInclusive(false);
    request.setFilterSet(filters);
    request.setGroupByDimensions(groupByDimensions);
    request.setEndDateInclusive(endTimeInclusive);
    LOG.info("Found [{}] time ranges to fetch data", startEndTimeRanges.size());
    for (Pair<Long, Long> timeRange : startEndTimeRanges) {
        LOG.info("Start Time [{}], End Time [{}] for anomaly analysis", new DateTime(timeRange.getFirst()), new DateTime(timeRange.getSecond()));
    }
    Set<TimeSeriesRow> timeSeriesRowSet = new HashSet<>();
    // TODO : replace this with Pinot MultiQuery Request
    for (Pair<Long, Long> startEndInterval : startEndTimeRanges) {
        DateTime startTime = new DateTime(startEndInterval.getFirst());
        DateTime endTime = new DateTime(startEndInterval.getSecond());
        request.setStart(startTime);
        request.setEnd(endTime);
        LOG.info("Fetching data with startTime: [{}], endTime: [{}], metricExpressions: [{}], timeGranularity: [{}]", startTime, endTime, metricExpressions, timeGranularity);
        try {
            LOG.debug("Executing {}", request);
            TimeSeriesResponse response = timeSeriesHandler.handle(request);
            timeSeriesRowSet.addAll(response.getRows());
        } catch (Exception e) {
            throw new JobExecutionException(e);
        }
    }
    List<TimeSeriesRow> timeSeriesRows = new ArrayList<>();
    timeSeriesRows.addAll(timeSeriesRowSet);
    return new TimeSeriesResponse(timeSeriesRows);
}

Example 4 with TimeSeriesResponse

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse in project pinot by linkedin.

the class TimeSeriesUtil method getTimeSeriesForAnomalyDetection.

/**
   * Returns the set of metric time series that are needed by the given anomaly function for detecting anomalies.
   *
   * The time granularity is the granularity of the function's collection, i.e., the buckets are not aggregated,
   * in order to increase the accuracy for detecting anomalies.
   *
   * @param anomalyFunctionSpec spec of the anomaly function
   * @param startEndTimeRanges the time ranges to retrieve the data for constructing the time series
   *
   * @return the data that is needed by the anomaly function for detecting anomalies.
   * @throws JobExecutionException
   * @throws ExecutionException
   */
public static Map<DimensionKey, MetricTimeSeries> getTimeSeriesForAnomalyDetection(AnomalyFunctionDTO anomalyFunctionSpec, List<Pair<Long, Long>> startEndTimeRanges) throws JobExecutionException, ExecutionException {
    String filterString = anomalyFunctionSpec.getFilters();
    Multimap<String, String> filters;
    if (StringUtils.isNotBlank(filterString)) {
        filters = ThirdEyeUtils.getFilterSet(filterString);
    } else {
        filters = HashMultimap.create();
    }
    List<String> groupByDimensions;
    String exploreDimensionString = anomalyFunctionSpec.getExploreDimensions();
    if (StringUtils.isNotBlank(exploreDimensionString)) {
        groupByDimensions = Arrays.asList(exploreDimensionString.trim().split(","));
    } else {
        groupByDimensions = Collections.emptyList();
    }
    TimeGranularity timeGranularity = new TimeGranularity(anomalyFunctionSpec.getBucketSize(), anomalyFunctionSpec.getBucketUnit());
    TimeSeriesResponse timeSeriesResponse = getTimeSeriesResponseImpl(anomalyFunctionSpec, startEndTimeRanges, timeGranularity, filters, groupByDimensions, false);
    try {
        Map<DimensionKey, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = TimeSeriesResponseConverter.toMap(timeSeriesResponse, Utils.getSchemaDimensionNames(anomalyFunctionSpec.getCollection()));
        return dimensionKeyMetricTimeSeriesMap;
    } catch (Exception e) {
        LOG.info("Failed to get schema dimensions for constructing dimension keys:", e.toString());
        return Collections.emptyMap();
    }
}

Example 5 with TimeSeriesResponse

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesResponse in project pinot by linkedin.

the class DashboardResource method getTimeSeriesData.

@GET
@Path(value = "/data/timeseries")
@Produces(MediaType.APPLICATION_JSON)
public String getTimeSeriesData(@QueryParam("dataset") String collection, @QueryParam("filters") String filterJson, @QueryParam("timeZone") @DefaultValue(DEFAULT_TIMEZONE_ID) String timeZone, @QueryParam("currentStart") Long start, @QueryParam("currentEnd") Long end, @QueryParam("aggTimeGranularity") String aggTimeGranularity, @QueryParam("metrics") String metricsJson, @QueryParam("dimensions") String groupByDimensions) throws Exception {
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(collection);
    // See {@link #getDashboardData} for the reason that the start and end time are stored in a
    // DateTime object with data's timezone.
    DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(collection);
    request.setStart(new DateTime(start, timeZoneForCollection));
    request.setEnd(new DateTime(end, timeZoneForCollection));
    if (groupByDimensions != null && !groupByDimensions.isEmpty()) {
        request.setGroupByDimensions(Arrays.asList(groupByDimensions.trim().split(",")));
    }
    if (filterJson != null && !filterJson.isEmpty()) {
        filterJson = URLDecoder.decode(filterJson, "UTF-8");
        request.setFilterSet(ThirdEyeUtils.convertToMultiMap(filterJson));
    }
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metricsJson, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY_INSTANCE.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getAggregationTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        request.setEndDateInclusive(true);
    }
    TimeSeriesHandler handler = new TimeSeriesHandler(queryCache);
    String jsonResponse = "";
    try {
        TimeSeriesResponse response = handler.handle(request);
        JSONObject timeseriesMap = new JSONObject();
        JSONArray timeValueArray = new JSONArray();
        TreeSet<String> keys = new TreeSet<>();
        TreeSet<Long> times = new TreeSet<>();
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            times.add(timeSeriesRow.getStart());
        }
        for (Long time : times) {
            timeValueArray.put(time);
        }
        timeseriesMap.put("time", timeValueArray);
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            for (TimeSeriesMetric metricTimeSeries : timeSeriesRow.getMetrics()) {
                String key = metricTimeSeries.getMetricName();
                if (timeSeriesRow.getDimensionNames() != null && timeSeriesRow.getDimensionNames().size() > 0) {
                    StringBuilder sb = new StringBuilder(key);
                    for (int idx = 0; idx < timeSeriesRow.getDimensionNames().size(); ++idx) {
                        sb.append("||").append(timeSeriesRow.getDimensionNames().get(idx));
                        sb.append("|").append(timeSeriesRow.getDimensionValues().get(idx));
                    }
                    key = sb.toString();
                }
                JSONArray valueArray;
                if (!timeseriesMap.has(key)) {
                    valueArray = new JSONArray();
                    timeseriesMap.put(key, valueArray);
                    keys.add(key);
                } else {
                    valueArray = timeseriesMap.getJSONArray(key);
                }
                valueArray.put(metricTimeSeries.getValue());
            }
        }
        JSONObject summaryMap = new JSONObject();
        summaryMap.put("currentStart", start);
        summaryMap.put("currentEnd", end);
        JSONObject jsonResponseObject = new JSONObject();
        jsonResponseObject.put("timeSeriesData", timeseriesMap);
        jsonResponseObject.put("keys", new JSONArray(keys));
        jsonResponseObject.put("summary", summaryMap);
        jsonResponse = jsonResponseObject.toString();
    } catch (Exception e) {
        throw e;
    }
    LOG.info("Response:{}", jsonResponse);
    return jsonResponse;
}