Examples with TimeSeriesMetric com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric used on opensource projects

Example 1 with TimeSeriesMetric

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric in project pinot by linkedin.

the class AnomaliesResource method getTimeSeriesData.

/**
   * Get timeseries for metric
   * @param collection
   * @param filters
   * @param start
   * @param end
   * @param aggTimeGranularity
   * @param metric
   * @return
   * @throws Exception
   */
private JSONObject getTimeSeriesData(String collection, Multimap<String, String> filters, Long start, Long end, String aggTimeGranularity, String metric) throws Exception {
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(collection);
    DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(collection);
    request.setStart(new DateTime(start, timeZoneForCollection));
    request.setEnd(new DateTime(end, timeZoneForCollection));
    request.setFilterSet(filters);
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metric, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getAggregationTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        request.setEndDateInclusive(true);
    }
    TimeSeriesHandler handler = new TimeSeriesHandler(CACHE_REGISTRY.getQueryCache());
    JSONObject jsonResponseObject = new JSONObject();
    try {
        TimeSeriesResponse response = handler.handle(request);
        JSONObject timeseriesMap = new JSONObject();
        JSONArray timeValueArray = new JSONArray();
        TreeSet<String> keys = new TreeSet<>();
        TreeSet<Long> times = new TreeSet<>();
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            times.add(timeSeriesRow.getStart());
        }
        for (Long time : times) {
            timeValueArray.put(time);
        }
        timeseriesMap.put("time", timeValueArray);
        for (int i = 0; i < response.getNumRows(); i++) {
            TimeSeriesRow timeSeriesRow = response.getRow(i);
            for (TimeSeriesMetric metricTimeSeries : timeSeriesRow.getMetrics()) {
                String key = metricTimeSeries.getMetricName();
                JSONArray valueArray;
                if (!timeseriesMap.has(key)) {
                    valueArray = new JSONArray();
                    timeseriesMap.put(key, valueArray);
                    keys.add(key);
                } else {
                    valueArray = timeseriesMap.getJSONArray(key);
                }
                valueArray.put(metricTimeSeries.getValue());
            }
        }
        JSONObject summaryMap = new JSONObject();
        summaryMap.put("currentStart", start);
        summaryMap.put("currentEnd", end);
        jsonResponseObject.put("timeSeriesData", timeseriesMap);
        jsonResponseObject.put("keys", new JSONArray(keys));
        jsonResponseObject.put("summary", summaryMap);
    } catch (Exception e) {
        throw e;
    }
    LOG.info("Response:{}", jsonResponseObject);
    return jsonResponseObject;
}

Example 2 with TimeSeriesMetric

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric in project pinot by linkedin.

the class TimeSeriesResponseParser method parseGroupByTimeDimensionResponse.

private void parseGroupByTimeDimensionResponse() {
    responseMap = ResponseParserUtils.createResponseMapByTimeAndDimension(response);
    Map<Integer, List<Double>> metricSums = ResponseParserUtils.getMetricSumsByTime(response);
    // group by time and dimension values
    Set<String> timeDimensionValues = new HashSet<>();
    timeDimensionValues.addAll(responseMap.keySet());
    Set<List<String>> dimensionValuesList = new HashSet<>();
    for (String timeDimensionValue : timeDimensionValues) {
        List<String> dimensionValues = ResponseParserUtils.extractDimensionValues(timeDimensionValue);
        dimensionValuesList.add(dimensionValues);
    }
    // group by dimension names (the 0th dimension, which is the time bucket, is skipped).
    List<String> groupKeyColumns = response.getGroupKeyColumns();
    List<String> dimensionNameList = new ArrayList<>(groupKeyColumns.size() - 1);
    for (int i = 1; i < groupKeyColumns.size(); ++i) {
        dimensionNameList.add(groupKeyColumns.get(i));
    }
    // other row
    List<TimeSeriesRow.Builder> otherBuilders = new ArrayList<>();
    List<double[]> otherMetrics = new ArrayList<>();
    boolean includeOther = false;
    // constructing an OTHER rows, 1 for each time bucket
    for (int timeBucketId = 0; timeBucketId < numTimeBuckets; timeBucketId++) {
        Range<DateTime> timeRange = ranges.get(timeBucketId);
        TimeSeriesRow.Builder builder = new TimeSeriesRow.Builder();
        builder.setStart(timeRange.lowerEndpoint());
        builder.setEnd(timeRange.upperEndpoint());
        builder.setDimensionNames(dimensionNameList);
        List<String> dimensionValues = new ArrayList(dimensionNameList.size());
        for (int i = 0; i < dimensionNameList.size(); ++i) {
            dimensionValues.add(OTHER);
        }
        builder.setDimensionValues(dimensionValues);
        otherBuilders.add(builder);
        double[] other = new double[numMetrics];
        Arrays.fill(other, 0);
        otherMetrics.add(other);
    }
    // else, we add the metric values to the OTHER row
    for (List<String> dimensionValues : dimensionValuesList) {
        List<TimeSeriesRow> thresholdRows = new ArrayList<>();
        for (int timeBucketId = 0; timeBucketId < numTimeBuckets; timeBucketId++) {
            Range<DateTime> timeRange = ranges.get(timeBucketId);
            // compute the time|dimension key
            String timeDimensionValue = ResponseParserUtils.computeTimeDimensionValues(timeBucketId, dimensionValues);
            ThirdEyeResponseRow responseRow = responseMap.get(timeDimensionValue);
            TimeSeriesRow.Builder builder = new TimeSeriesRow.Builder();
            builder.setStart(timeRange.lowerEndpoint());
            builder.setEnd(timeRange.upperEndpoint());
            builder.setDimensionNames(dimensionNameList);
            builder.setDimensionValues(dimensionValues);
            addMetric(responseRow, builder);
            TimeSeriesRow row = builder.build();
            thresholdRows.add(row);
        }
        // check if rows pass threshold
        boolean passedThreshold = false;
        for (int timeBucketId = 0; timeBucketId < numTimeBuckets; timeBucketId++) {
            if (checkMetricSums(thresholdRows.get(timeBucketId), metricSums.get(timeBucketId))) {
                passedThreshold = true;
                break;
            }
        }
        // if any of the cells of a contributor row passes threshold, add all those cells
        if (passedThreshold && !dimensionValues.contains(OTHER)) {
            rows.addAll(thresholdRows);
        } else {
            // else that row of cells goes into OTHER
            includeOther = true;
            for (int timeBucketId = 0; timeBucketId < numTimeBuckets; timeBucketId++) {
                TimeSeriesRow row = thresholdRows.get(timeBucketId);
                List<TimeSeriesMetric> metrics = row.getMetrics();
                for (int i = 0; i < metrics.size(); i++) {
                    TimeSeriesMetric metricToAdd = metrics.get(i);
                    otherMetrics.get(timeBucketId)[i] += metricToAdd.getValue();
                }
            }
        }
    }
    // create other row using the other sums
    if (includeOther) {
        for (int timeBucketId = 0; timeBucketId < numTimeBuckets; timeBucketId++) {
            Builder otherBuilder = otherBuilders.get(timeBucketId);
            double[] other = otherMetrics.get(timeBucketId);
            for (int i = 0; i < numMetrics; i++) {
                otherBuilder.addMetric(metricFunctions.get(i).getMetricName(), other[i]);
            }
            rows.add(otherBuilder.build());
        }
    }
}

Example 3 with TimeSeriesMetric

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric in project pinot by linkedin.

the class TimeSeriesTest method main.

public static void main(String[] args) throws Exception {
    // = System.class.get.getResource("log4j.properties");
    URL resource = null;
    if (resource == null) {
        resource = TimeSeriesHandler.class.getClassLoader().getResource("logback.x");
    }
    // TODO
    PinotThirdEyeClient pinotThirdEyeClient = PinotThirdEyeClient.getDefaultTestClient();
    // make
    // this
    // configurable
    QueryCache queryCache = new QueryCache(pinotThirdEyeClient, Executors.newFixedThreadPool(10));
    TimeSeriesRequest[] requests = new TimeSeriesRequest[] { generateGroupByTimeRequest() };
    for (TimeSeriesRequest timeSeriesRequest : requests) {
        try {
            TimeSeriesHandler handler = new TimeSeriesHandler(queryCache);
            long start = System.currentTimeMillis();
            TimeSeriesResponse response = handler.handle(timeSeriesRequest);
            long end = System.currentTimeMillis();
            System.out.println("Time taken:" + (end - start));
            for (TimeSeriesMetric metric : response.getRow(0).getMetrics()) {
                System.out.print(metric.getMetricName() + "\t\t");
            }
            System.out.println();
            for (int i = 0; i < response.getNumRows(); i++) {
                System.out.println(response.getRow(i));
            }
        } catch (Exception e) {
            System.out.println("Request failed: " + timeSeriesRequest);
            e.printStackTrace();
            System.exit(-1);
        }
    }
    System.out.println("No exceptions encountered during testing... but you still need to check for data quality!");
    System.exit(0);
}

Example 4 with TimeSeriesMetric

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric in project pinot by linkedin.

the class TestTimeSeriesResponseUtils method createMapProviderArgs.

/**
   * return type: String testName, TimeSeriesResponse, List<String> dimensions, Map<DimensionKey,
   * MetricTimeSeries>.
   */
public Object[] createMapProviderArgs(String testName, boolean groupByDimension, boolean groupTimeSeriesMetricsIntoRow) {
    List<String> dimensions = Arrays.asList("dim1", "dim2", "dim3", "all");
    // appended to each
    List<String> dimensionValueSuffixes = Arrays.asList("_a", "_b", "_c");
    // dimension
    List<MetricFunction> metricFunctions = createSumFunctions("m1", "m2", "m3");
    ConversionDataGenerator dataGenerator = new ConversionDataGenerator(dimensions, metricFunctions);
    for (long hoursSinceEpoch = 0; hoursSinceEpoch < 50; hoursSinceEpoch++) {
        DateTime start = new DateTime(hoursSinceEpoch);
        DateTime end = start.plusHours(1);
        for (String dimension : (groupByDimension ? dimensions : Collections.<String>singleton("all"))) {
            for (String dimensionValueSuffix : (groupByDimension ? dimensionValueSuffixes : Collections.<String>singleton("all"))) {
                String dimensionValue;
                if (groupByDimension) {
                    dimensionValue = dimension + dimensionValueSuffix;
                } else {
                    dimensionValue = "all";
                }
                List<TimeSeriesMetric> timeSeriesMetrics = new ArrayList<>();
                for (MetricFunction metricFunction : metricFunctions) {
                    Double value = (double) (Objects.hash(start, end, dimension, dimensionValue, metricFunction.toString()) % // doesn't matter, the test is that values are
                    1000);
                    // consistent between data
                    // structures.
                    TimeSeriesMetric timeSeriesMetric = new TimeSeriesMetric(metricFunction.getMetricName(), value);
                    timeSeriesMetrics.add(timeSeriesMetric);
                }
                if (groupTimeSeriesMetricsIntoRow) {
                    // add them all at once
                    dataGenerator.addEntry(Arrays.asList(dimension), Arrays.asList(dimensionValue), start, end, timeSeriesMetrics.toArray(new TimeSeriesMetric[timeSeriesMetrics.size()]));
                } else {
                    // add them individually (one metric per row)
                    for (TimeSeriesMetric timeSeriesMetric : timeSeriesMetrics) {
                        dataGenerator.addEntry(Arrays.asList(dimension), Arrays.asList(dimensionValue), start, end, timeSeriesMetric);
                    }
                }
            }
        }
    }
    Object[] dimensionGroupByArgs = new Object[] { testName, dataGenerator.getResponse(), dimensions, dataGenerator.getMap() };
    return dimensionGroupByArgs;
}

Example 5 with TimeSeriesMetric

use of com.linkedin.thirdeye.client.timeseries.TimeSeriesRow.TimeSeriesMetric in project pinot by linkedin.

the class TimeSeriesResponseParser method checkMetricSums.

private boolean checkMetricSums(TimeSeriesRow row, List<Double> metricSums) {
    List<TimeSeriesMetric> metrics = row.getMetrics();
    for (int i = 0; i < metrics.size(); i++) {
        TimeSeriesMetric metric = metrics.get(i);
        double sum = 0;
        if (metricSums != null) {
            sum = metricSums.get(i);
        }
        if (metric.getValue() > metricThresholds.get(metric.getMetricName()) * sum) {
            return true;
        }
    }
    return false;
}