Examples with MetricFunction com.linkedin.thirdeye.client.MetricFunction used on opensource projects

Example 1 with MetricFunction

use of com.linkedin.thirdeye.client.MetricFunction in project pinot by linkedin.

the class TimeOnTimeResponseParser method addMetric.

private void addMetric(ThirdEyeResponseRow baselineRow, ThirdEyeResponseRow currentRow, Builder builder) {
    List<MetricFunction> metricFunctions = baselineResponse.getMetricFunctions();
    for (int i = 0; i < metricFunctions.size(); i++) {
        MetricFunction metricFunction = metricFunctions.get(i);
        double baselineValue = 0;
        if (baselineRow != null) {
            baselineValue = baselineRow.getMetrics().get(i);
        }
        double currentValue = 0;
        if (currentRow != null) {
            currentValue = currentRow.getMetrics().get(i);
        }
        builder.addMetric(metricFunction.getMetricName(), baselineValue, currentValue);
    }
}

Example 2 with MetricFunction

use of com.linkedin.thirdeye.client.MetricFunction in project pinot by linkedin.

the class TimeOnTimeComparisonHandler method computeDerivedMetrics.

private void computeDerivedMetrics(TimeOnTimeComparisonRequest comparisonRequest, List<Row> rows) throws Exception {
    // compute list of derived expressions
    List<MetricFunction> metricFunctionsFromExpressions = Utils.computeMetricFunctionsFromExpressions(comparisonRequest.getMetricExpressions());
    Set<String> metricNameSet = new HashSet<>();
    for (MetricFunction function : metricFunctionsFromExpressions) {
        metricNameSet.add(function.getMetricName());
    }
    List<MetricExpression> derivedMetricExpressions = new ArrayList<>();
    for (MetricExpression expression : comparisonRequest.getMetricExpressions()) {
        if (!metricNameSet.contains(expression.getExpressionName())) {
            derivedMetricExpressions.add(expression);
        }
    }
    // add metric expressions
    if (derivedMetricExpressions.size() > 0) {
        Map<String, Double> baselineValueContext = new HashMap<>();
        Map<String, Double> currentValueContext = new HashMap<>();
        for (Row row : rows) {
            baselineValueContext.clear();
            currentValueContext.clear();
            List<Metric> metrics = row.getMetrics();
            // baseline value
            for (Metric metric : metrics) {
                baselineValueContext.put(metric.getMetricName(), metric.getBaselineValue());
                currentValueContext.put(metric.getMetricName(), metric.getCurrentValue());
            }
            for (MetricExpression expression : derivedMetricExpressions) {
                String derivedMetricExpression = expression.getExpression();
                double derivedMetricBaselineValue = MetricExpression.evaluateExpression(derivedMetricExpression, baselineValueContext);
                if (Double.isInfinite(derivedMetricBaselineValue) || Double.isNaN(derivedMetricBaselineValue)) {
                    derivedMetricBaselineValue = 0;
                }
                double currentMetricBaselineValue = MetricExpression.evaluateExpression(derivedMetricExpression, currentValueContext);
                if (Double.isInfinite(currentMetricBaselineValue) || Double.isNaN(currentMetricBaselineValue)) {
                    currentMetricBaselineValue = 0;
                }
                row.getMetrics().add(new Metric(expression.getExpressionName(), derivedMetricBaselineValue, currentMetricBaselineValue));
            }
        }
    }
}

Example 3 with MetricFunction

use of com.linkedin.thirdeye.client.MetricFunction in project pinot by linkedin.

the class Utils method getFilters.

public static Map<String, List<String>> getFilters(QueryCache queryCache, String collection, String requestReference, List<String> dimensions, DateTime start, DateTime end) throws Exception {
    MetricFunction metricFunction = new MetricFunction(MetricAggFunction.COUNT, "*");
    List<ThirdEyeRequest> requests = generateRequests(collection, requestReference, metricFunction, dimensions, start, end);
    Map<ThirdEyeRequest, Future<ThirdEyeResponse>> queryResultMap = queryCache.getQueryResultsAsync(requests);
    Map<String, List<String>> result = new HashMap<>();
    for (Map.Entry<ThirdEyeRequest, Future<ThirdEyeResponse>> entry : queryResultMap.entrySet()) {
        ThirdEyeRequest request = entry.getKey();
        String dimension = request.getGroupBy().get(0);
        ThirdEyeResponse thirdEyeResponse = entry.getValue().get();
        int n = thirdEyeResponse.getNumRowsFor(metricFunction);
        List<String> values = new ArrayList<>();
        for (int i = 0; i < n; i++) {
            Map<String, String> row = thirdEyeResponse.getRow(metricFunction, i);
            String dimensionValue = row.get(dimension);
            values.add(dimensionValue);
        }
        result.put(dimension, values);
    }
    return result;
}

Example 4 with MetricFunction

use of com.linkedin.thirdeye.client.MetricFunction in project pinot by linkedin.

the class Utils method generateRequests.

public static List<ThirdEyeRequest> generateRequests(String collection, String requestReference, MetricFunction metricFunction, List<String> dimensions, DateTime start, DateTime end) {
    List<ThirdEyeRequest> requests = new ArrayList<>();
    for (String dimension : dimensions) {
        ThirdEyeRequestBuilder requestBuilder = new ThirdEyeRequestBuilder();
        requestBuilder.setCollection(collection);
        List<MetricFunction> metricFunctions = Arrays.asList(metricFunction);
        requestBuilder.setMetricFunctions(metricFunctions);
        requestBuilder.setStartTimeInclusive(start);
        requestBuilder.setEndTimeExclusive(end);
        requestBuilder.setGroupBy(dimension);
        ThirdEyeRequest request = requestBuilder.build(requestReference);
        requests.add(request);
    }
    return requests;
}

Example 5 with MetricFunction

use of com.linkedin.thirdeye.client.MetricFunction in project pinot by linkedin.

the class HeatMapViewHandler method process.

@Override
public HeatMapViewResponse process(HeatMapViewRequest request) throws Exception {
    // query 1 for everything from baseline start to baseline end
    // query 2 for everything from current start to current end
    // for each dimension group by top 100
    // query 1 for everything from baseline start to baseline end
    // query for everything from current start to current end
    List<String> expressionNames = new ArrayList<>();
    Map<String, String> metricExpressions = new HashMap<>();
    Set<String> metricOrExpressionNames = new HashSet<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
        metricExpressions.put(expression.getExpressionName(), expression.getExpression());
        metricOrExpressionNames.add(expression.getExpressionName());
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        for (MetricFunction function : metricFunctions) {
            metricOrExpressionNames.add(function.getMetricName());
        }
    }
    Map<String, HeatMap.Builder> data = new HashMap<>();
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    List<String> groupByDimensions = comparisonRequest.getGroupByDimensions();
    final TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    // we are tracking per dimension, to validate that its the same for each dimension
    Map<String, Map<String, Double>> baselineTotalPerMetricAndDimension = new HashMap<>();
    Map<String, Map<String, Double>> currentTotalPerMetricAndDimension = new HashMap<>();
    for (String metricOrExpressionName : metricOrExpressionNames) {
        Map<String, Double> baselineTotalMap = new HashMap<>();
        Map<String, Double> currentTotalMap = new HashMap<>();
        baselineTotalPerMetricAndDimension.put(metricOrExpressionName, baselineTotalMap);
        currentTotalPerMetricAndDimension.put(metricOrExpressionName, currentTotalMap);
        for (String dimension : groupByDimensions) {
            baselineTotalMap.put(dimension, 0d);
            currentTotalMap.put(dimension, 0d);
        }
    }
    List<Future<TimeOnTimeComparisonResponse>> timeOnTimeComparisonResponsesFutures = getTimeOnTimeComparisonResponses(groupByDimensions, comparisonRequest, handler);
    for (int groupByDimensionId = 0; groupByDimensionId < groupByDimensions.size(); groupByDimensionId++) {
        String groupByDimension = groupByDimensions.get(groupByDimensionId);
        TimeOnTimeComparisonResponse response = timeOnTimeComparisonResponsesFutures.get(groupByDimensionId).get();
        int numRows = response.getNumRows();
        for (int i = 0; i < numRows; i++) {
            Row row = response.getRow(i);
            String dimensionValue = row.getDimensionValue();
            Map<String, Metric> metricMap = new HashMap<>();
            for (Metric metric : row.getMetrics()) {
                metricMap.put(metric.getMetricName(), metric);
            }
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                // update the baselineTotal and current total
                Map<String, Double> baselineTotalMap = baselineTotalPerMetricAndDimension.get(metricName);
                Map<String, Double> currentTotalMap = currentTotalPerMetricAndDimension.get(metricName);
                baselineTotalMap.put(groupByDimension, baselineTotalMap.get(groupByDimension) + metric.getBaselineValue());
                currentTotalMap.put(groupByDimension, currentTotalMap.get(groupByDimension) + metric.getCurrentValue());
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                String dataKey = metricName + "." + groupByDimension;
                HeatMap.Builder heatMapBuilder = data.get(dataKey);
                if (heatMapBuilder == null) {
                    heatMapBuilder = new HeatMap.Builder(groupByDimension);
                    data.put(dataKey, heatMapBuilder);
                }
                MetricDataset metricDataset = new MetricDataset(metricName, comparisonRequest.getCollectionName());
                MetricConfigDTO metricConfig = CACHE_REGISTRY.getMetricConfigCache().get(metricDataset);
                if (StringUtils.isNotBlank(metricConfig.getCellSizeExpression())) {
                    String metricExpression = metricExpressions.get(metricName);
                    String[] tokens = metricExpression.split(RATIO_SEPARATOR);
                    String numerator = tokens[0];
                    String denominator = tokens[1];
                    Metric numeratorMetric = metricMap.get(numerator);
                    Metric denominatorMetric = metricMap.get(denominator);
                    Double numeratorBaseline = numeratorMetric == null ? 0 : numeratorMetric.getBaselineValue();
                    Double numeratorCurrent = numeratorMetric == null ? 0 : numeratorMetric.getCurrentValue();
                    Double denominatorBaseline = denominatorMetric == null ? 0 : denominatorMetric.getBaselineValue();
                    Double denominatorCurrent = denominatorMetric == null ? 0 : denominatorMetric.getCurrentValue();
                    Map<String, Double> context = new HashMap<>();
                    context.put(numerator, numeratorCurrent);
                    context.put(denominator, denominatorCurrent);
                    String cellSizeExpression = metricConfig.getCellSizeExpression();
                    Double cellSize = MetricExpression.evaluateExpression(cellSizeExpression, context);
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue(), cellSize, cellSizeExpression, numeratorBaseline, denominatorBaseline, numeratorCurrent, denominatorCurrent);
                } else {
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue());
                }
            }
        }
    }
    ResponseSchema schema = new ResponseSchema();
    String[] columns = HeatMapCell.columns();
    for (int i = 0; i < columns.length; i++) {
        String column = columns[i];
        schema.add(column, i);
    }
    Info summary = new Info();
    Map<String, GenericResponse> heatMapViewResponseData = new HashMap<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        Double baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        Double currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        // check if its derived
        if (metricFunctions.size() > 1) {
            Map<String, Double> baselineContext = new HashMap<>();
            Map<String, Double> currentContext = new HashMap<>();
            for (String metricOrExpression : metricOrExpressionNames) {
                baselineContext.put(metricOrExpression, baselineTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
                currentContext.put(metricOrExpression, currentTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
            }
            baselineTotal = MetricExpression.evaluateExpression(expression, baselineContext);
            currentTotal = MetricExpression.evaluateExpression(expression, currentContext);
        } else {
            baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
            currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        }
        summary.addSimpleField("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
        summary.addSimpleField("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
        summary.addSimpleField("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
        summary.addSimpleField("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
        summary.addSimpleField("baselineTotal", HeatMapCell.format(baselineTotal));
        summary.addSimpleField("currentTotal", HeatMapCell.format(currentTotal));
        summary.addSimpleField("deltaChange", HeatMapCell.format(currentTotal - baselineTotal));
        summary.addSimpleField("deltaPercentage", HeatMapCell.format((currentTotal - baselineTotal) * 100.0 / baselineTotal));
    }
    for (Entry<String, HeatMap.Builder> entry : data.entrySet()) {
        String dataKey = entry.getKey();
        GenericResponse heatMapResponse = new GenericResponse();
        List<String[]> heatMapResponseData = new ArrayList<>();
        HeatMap.Builder builder = entry.getValue();
        HeatMap heatMap = builder.build();
        for (HeatMapCell cell : heatMap.heatMapCells) {
            String[] newRowData = cell.toArray();
            heatMapResponseData.add(newRowData);
        }
        heatMapResponse.setSchema(schema);
        heatMapResponse.setResponseData(heatMapResponseData);
        heatMapViewResponseData.put(dataKey, heatMapResponse);
    }
    HeatMapViewResponse heatMapViewResponse = new HeatMapViewResponse();
    heatMapViewResponse.setMetrics(expressionNames);
    heatMapViewResponse.setDimensions(groupByDimensions);
    heatMapViewResponse.setData(heatMapViewResponseData);
    heatMapViewResponse.setMetricExpression(metricExpressions);
    heatMapViewResponse.setSummary(summary);
    return heatMapViewResponse;
}