Examples with MetricExpression com.linkedin.thirdeye.client.MetricExpression used on opensource projects

Example 21 with MetricExpression

use of com.linkedin.thirdeye.client.MetricExpression in project pinot by linkedin.

the class EmailHelper method getContributorDataForDataReport.

public static ContributorViewResponse getContributorDataForDataReport(String collection, String metric, List<String> dimensions, AlertConfigBean.COMPARE_MODE compareMode, long offsetDelayMillis, boolean intraday) throws Exception {
    long baselineOffset = getBaselineOffset(compareMode);
    ContributorViewRequest request = new ContributorViewRequest();
    request.setCollection(collection);
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metric, MetricAggFunction.SUM, collection);
    request.setMetricExpressions(metricExpressions);
    long currentEnd = System.currentTimeMillis();
    long maxDataTime = collectionMaxDataTimeCache.get(collection);
    if (currentEnd > maxDataTime) {
        currentEnd = maxDataTime;
    }
    // align to nearest hour
    currentEnd = (currentEnd - (currentEnd % HOUR_MILLIS)) - offsetDelayMillis;
    String aggTimeGranularity = "HOURS";
    long currentStart = currentEnd - DAY_MILLIS;
    // intraday option
    if (intraday) {
        DateTimeZone timeZone = DateTimeZone.forTimeZone(AlertTaskRunnerV2.DEFAULT_TIME_ZONE);
        DateTime endDate = new DateTime(currentEnd, timeZone);
        DateTime intraDayStartTime = new DateTime(endDate.toString().split("T")[0], timeZone);
        if (intraDayStartTime.getMillis() != currentEnd) {
            currentStart = intraDayStartTime.getMillis();
        }
    }
    DatasetConfigDTO datasetConfigDTO = datasetConfigManager.findByDataset(collection);
    if (datasetConfigDTO != null && TimeUnit.DAYS.equals(datasetConfigDTO.getTimeUnit())) {
        aggTimeGranularity = datasetConfigDTO.getTimeUnit().name();
        currentEnd = currentEnd - (currentEnd % DAY_MILLIS);
        currentStart = currentEnd - WEEK_MILLIS;
    }
    long baselineStart = currentStart - baselineOffset;
    long baselineEnd = currentEnd - baselineOffset;
    String timeZone = datasetConfigDTO.getTimezone();
    request.setBaselineStart(new DateTime(baselineStart, DateTimeZone.forID(timeZone)));
    request.setBaselineEnd(new DateTime(baselineEnd, DateTimeZone.forID(timeZone)));
    request.setCurrentStart(new DateTime(currentStart, DateTimeZone.forID(timeZone)));
    request.setCurrentEnd(new DateTime(currentEnd, DateTimeZone.forID(timeZone)));
    request.setTimeGranularity(Utils.getAggregationTimeGranularity(aggTimeGranularity, collection));
    request.setGroupByDimensions(dimensions);
    ContributorViewHandler handler = new ContributorViewHandler(queryCache);
    return handler.process(request);
}

Example 22 with MetricExpression

use of com.linkedin.thirdeye.client.MetricExpression in project pinot by linkedin.

the class TimeSeriesUtil method getTimeSeriesResponseImpl.

private static TimeSeriesResponse getTimeSeriesResponseImpl(AnomalyFunctionDTO anomalyFunctionSpec, List<Pair<Long, Long>> startEndTimeRanges, TimeGranularity timeGranularity, Multimap<String, String> filters, List<String> groupByDimensions, boolean endTimeInclusive) throws JobExecutionException, ExecutionException {
    TimeSeriesHandler timeSeriesHandler = new TimeSeriesHandler(ThirdEyeCacheRegistry.getInstance().getQueryCache());
    // Seed request with top-level...
    TimeSeriesRequest request = new TimeSeriesRequest();
    request.setCollectionName(anomalyFunctionSpec.getCollection());
    // TODO: Check low level support for multiple metrics retrieval
    String metricsToRetrieve = String.join(",", anomalyFunctionSpec.getMetrics());
    List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metricsToRetrieve, anomalyFunctionSpec.getMetricFunction(), anomalyFunctionSpec.getCollection());
    request.setMetricExpressions(metricExpressions);
    request.setAggregationTimeGranularity(timeGranularity);
    request.setEndDateInclusive(false);
    request.setFilterSet(filters);
    request.setGroupByDimensions(groupByDimensions);
    request.setEndDateInclusive(endTimeInclusive);
    LOG.info("Found [{}] time ranges to fetch data", startEndTimeRanges.size());
    for (Pair<Long, Long> timeRange : startEndTimeRanges) {
        LOG.info("Start Time [{}], End Time [{}] for anomaly analysis", new DateTime(timeRange.getFirst()), new DateTime(timeRange.getSecond()));
    }
    Set<TimeSeriesRow> timeSeriesRowSet = new HashSet<>();
    // TODO : replace this with Pinot MultiQuery Request
    for (Pair<Long, Long> startEndInterval : startEndTimeRanges) {
        DateTime startTime = new DateTime(startEndInterval.getFirst());
        DateTime endTime = new DateTime(startEndInterval.getSecond());
        request.setStart(startTime);
        request.setEnd(endTime);
        LOG.info("Fetching data with startTime: [{}], endTime: [{}], metricExpressions: [{}], timeGranularity: [{}]", startTime, endTime, metricExpressions, timeGranularity);
        try {
            LOG.debug("Executing {}", request);
            TimeSeriesResponse response = timeSeriesHandler.handle(request);
            timeSeriesRowSet.addAll(response.getRows());
        } catch (Exception e) {
            throw new JobExecutionException(e);
        }
    }
    List<TimeSeriesRow> timeSeriesRows = new ArrayList<>();
    timeSeriesRows.addAll(timeSeriesRowSet);
    return new TimeSeriesResponse(timeSeriesRows);
}

Example 23 with MetricExpression

use of com.linkedin.thirdeye.client.MetricExpression in project pinot by linkedin.

the class TimeSeriesHandler method computeDerivedMetrics.

private void computeDerivedMetrics(TimeSeriesRequest timeSeriesRequest, List<TimeSeriesRow> rows) throws Exception {
    // compute list of derived expressions
    List<MetricFunction> metricFunctionsFromExpressions = Utils.computeMetricFunctionsFromExpressions(timeSeriesRequest.getMetricExpressions());
    Set<String> metricNameSet = new HashSet<>();
    for (MetricFunction function : metricFunctionsFromExpressions) {
        metricNameSet.add(function.getMetricName());
    }
    List<MetricExpression> derivedMetricExpressions = new ArrayList<>();
    for (MetricExpression expression : timeSeriesRequest.getMetricExpressions()) {
        if (!metricNameSet.contains(expression.getExpressionName())) {
            derivedMetricExpressions.add(expression);
        }
    }
    // add metric expressions
    if (derivedMetricExpressions.size() > 0) {
        Map<String, Double> valueContext = new HashMap<>();
        for (TimeSeriesRow row : rows) {
            valueContext.clear();
            List<TimeSeriesMetric> metrics = row.getMetrics();
            // baseline value
            for (TimeSeriesMetric metric : metrics) {
                valueContext.put(metric.getMetricName(), metric.getValue());
            }
            for (MetricExpression expression : derivedMetricExpressions) {
                String derivedMetricExpression = expression.getExpression();
                double derivedMetricValue = MetricExpression.evaluateExpression(derivedMetricExpression, valueContext);
                if (Double.isInfinite(derivedMetricValue) || Double.isNaN(derivedMetricValue)) {
                    derivedMetricValue = 0;
                }
                row.getMetrics().add(new TimeSeriesMetric(expression.getExpressionName(), derivedMetricValue));
            }
        }
    }
}

Example 24 with MetricExpression

use of com.linkedin.thirdeye.client.MetricExpression in project pinot by linkedin.

the class TimeSeriesHandler method createThirdEyeRequest.

private static ThirdEyeRequest createThirdEyeRequest(String requestReference, TimeSeriesRequest timeSeriesRequest, DateTime start, DateTime end) {
    ThirdEyeRequestBuilder requestBuilder = ThirdEyeRequest.newBuilder();
    requestBuilder.setCollection(timeSeriesRequest.getCollectionName());
    requestBuilder.setStartTimeInclusive(start);
    requestBuilder.setEndTimeExclusive(end);
    requestBuilder.setFilterSet(timeSeriesRequest.getFilterSet());
    requestBuilder.addGroupBy(timeSeriesRequest.getGroupByDimensions());
    requestBuilder.setGroupByTimeGranularity(timeSeriesRequest.getAggregationTimeGranularity());
    List<MetricExpression> metricExpressions = timeSeriesRequest.getMetricExpressions();
    List<MetricFunction> metricFunctionsFromExpressions = Utils.computeMetricFunctionsFromExpressions(metricExpressions);
    requestBuilder.setMetricFunctions(metricFunctionsFromExpressions);
    return requestBuilder.build(requestReference);
}

Example 25 with MetricExpression

use of com.linkedin.thirdeye.client.MetricExpression in project pinot by linkedin.

the class SummaryResource method buildSummary.

@GET
@Path(value = "/summary/autoDimensionOrder")
@Produces(MediaType.APPLICATION_JSON)
public String buildSummary(@QueryParam("dataset") String collection, @QueryParam("metric") String metric, @QueryParam("currentStart") Long currentStartInclusive, @QueryParam("currentEnd") Long currentEndExclusive, @QueryParam("baselineStart") Long baselineStartInclusive, @QueryParam("baselineEnd") Long baselineEndExclusive, @QueryParam("dimensions") String groupByDimensions, @QueryParam("summarySize") int summarySize, @QueryParam("topDimensions") @DefaultValue(DEFAULT_TOP_DIMENSIONS) int topDimensions, @QueryParam("hierarchies") @DefaultValue(DEFAULT_HIERARCHIES) String hierarchiesPayload, @QueryParam("oneSideError") @DefaultValue(DEFAULT_ONE_SIDE_ERROR) boolean doOneSideError, @QueryParam("timeZone") @DefaultValue(DEFAULT_TIMEZONE_ID) String timeZone) throws Exception {
    if (summarySize < 1)
        summarySize = 1;
    SummaryResponse response = null;
    try {
        List<MetricExpression> metricExpressions = Utils.convertToMetricExpressions(metric, MetricAggFunction.SUM, collection);
        OLAPDataBaseClient olapClient = new PinotThirdEyeSummaryClient(CACHE_REGISTRY_INSTANCE.getQueryCache());
        olapClient.setCollection(collection);
        olapClient.setMetricExpression(metricExpressions.get(0));
        olapClient.setCurrentStartInclusive(new DateTime(currentStartInclusive, DateTimeZone.forID(timeZone)));
        olapClient.setCurrentEndExclusive(new DateTime(currentEndExclusive, DateTimeZone.forID(timeZone)));
        olapClient.setBaselineStartInclusive(new DateTime(baselineStartInclusive, DateTimeZone.forID(timeZone)));
        olapClient.setBaselineEndExclusive(new DateTime(baselineEndExclusive, DateTimeZone.forID(timeZone)));
        Dimensions dimensions;
        if (groupByDimensions == null || groupByDimensions.length() == 0 || groupByDimensions.equals("undefined")) {
            dimensions = new Dimensions(Utils.getSchemaDimensionNames(collection));
        } else {
            dimensions = new Dimensions(Arrays.asList(groupByDimensions.trim().split(",")));
        }
        List<List<String>> hierarchies = OBJECT_MAPPER.readValue(hierarchiesPayload, new TypeReference<List<List<String>>>() {
        });
        Cube cube = new Cube();
        cube.buildWithAutoDimensionOrder(olapClient, dimensions, topDimensions, hierarchies);
        Summary summary = new Summary(cube);
        response = summary.computeSummary(summarySize, doOneSideError, topDimensions);
        response.setMetricName(metric);
    } catch (Exception e) {
        LOG.error("Exception while generating difference summary", e);
        response = SummaryResponse.buildNotAvailableResponse();
        response.setMetricName(metric);
    }
    return OBJECT_MAPPER.writeValueAsString(response);
}