Examples with TimeOnTimeComparisonRequest com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest used on opensource projects

Example 1 with TimeOnTimeComparisonRequest

use of com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest in project pinot by linkedin.

the class EmailHelper method getData.

/**
   * Generate and send request to retrieve chart data. If the request window is too small, the graph
   * data retrieved has default window sizes based on time granularity and ending at the defined
   * endpoint: <br/> <ul> <li>DAYS: 7</li> <li>HOURS: 24</li> </ul>
   *
   * @param bucketGranularity
   *
   * @throws JobExecutionException
   */
public static TimeOnTimeComparisonResponse getData(TimeOnTimeComparisonHandler timeOnTimeComparisonHandler, EmailConfigurationDTO config, DateTime start, final DateTime end, long baselinePeriodMillis, TimeGranularity bucketGranularity) throws JobExecutionException {
    start = calculateGraphDataStart(start, end, bucketGranularity);
    try {
        TimeOnTimeComparisonRequest comparisonRequest = new TimeOnTimeComparisonRequest();
        comparisonRequest.setCollectionName(config.getCollection());
        comparisonRequest.setBaselineStart(start.minus(baselinePeriodMillis));
        comparisonRequest.setBaselineEnd(end.minus(baselinePeriodMillis));
        comparisonRequest.setCurrentStart(start);
        comparisonRequest.setCurrentEnd(end);
        comparisonRequest.setEndDateInclusive(true);
        List<MetricExpression> metricExpressions = new ArrayList<>();
        MetricExpression expression = new MetricExpression(config.getMetric());
        metricExpressions.add(expression);
        comparisonRequest.setMetricExpressions(metricExpressions);
        comparisonRequest.setAggregationTimeGranularity(bucketGranularity);
        LOG.debug("Starting...");
        TimeOnTimeComparisonResponse response = timeOnTimeComparisonHandler.handle(comparisonRequest);
        LOG.debug("Done!");
        return response;
    } catch (Exception e) {
        throw new JobExecutionException(e);
    }
}

Example 2 with TimeOnTimeComparisonRequest

use of com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest in project pinot by linkedin.

the class HeatMapViewHandler method generateTimeOnTimeComparisonRequest.

private TimeOnTimeComparisonRequest generateTimeOnTimeComparisonRequest(HeatMapViewRequest request) throws Exception {
    TimeOnTimeComparisonRequest comparisonRequest = new TimeOnTimeComparisonRequest();
    String collection = request.getCollection();
    DateTime baselineStart = request.getBaselineStart();
    DateTime baselineEnd = request.getBaselineEnd();
    DateTime currentStart = request.getCurrentStart();
    DateTime currentEnd = request.getCurrentEnd();
    comparisonRequest.setEndDateInclusive(false);
    Multimap<String, String> filters = request.getFilters();
    List<String> dimensionsToGroupBy = Utils.getDimensionsToGroupBy(collection, filters);
    List<MetricExpression> metricExpressions = request.getMetricExpressions();
    comparisonRequest.setCollectionName(collection);
    comparisonRequest.setBaselineStart(baselineStart);
    comparisonRequest.setBaselineEnd(baselineEnd);
    comparisonRequest.setCurrentStart(currentStart);
    comparisonRequest.setCurrentEnd(currentEnd);
    comparisonRequest.setGroupByDimensions(dimensionsToGroupBy);
    comparisonRequest.setFilterSet(filters);
    comparisonRequest.setMetricExpressions(metricExpressions);
    comparisonRequest.setAggregationTimeGranularity(null);
    return comparisonRequest;
}

Example 3 with TimeOnTimeComparisonRequest

use of com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest in project pinot by linkedin.

the class HeatMapViewHandler method process.

@Override
public HeatMapViewResponse process(HeatMapViewRequest request) throws Exception {
    // query 1 for everything from baseline start to baseline end
    // query 2 for everything from current start to current end
    // for each dimension group by top 100
    // query 1 for everything from baseline start to baseline end
    // query for everything from current start to current end
    List<String> expressionNames = new ArrayList<>();
    Map<String, String> metricExpressions = new HashMap<>();
    Set<String> metricOrExpressionNames = new HashSet<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
        metricExpressions.put(expression.getExpressionName(), expression.getExpression());
        metricOrExpressionNames.add(expression.getExpressionName());
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        for (MetricFunction function : metricFunctions) {
            metricOrExpressionNames.add(function.getMetricName());
        }
    }
    Map<String, HeatMap.Builder> data = new HashMap<>();
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    List<String> groupByDimensions = comparisonRequest.getGroupByDimensions();
    final TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    // we are tracking per dimension, to validate that its the same for each dimension
    Map<String, Map<String, Double>> baselineTotalPerMetricAndDimension = new HashMap<>();
    Map<String, Map<String, Double>> currentTotalPerMetricAndDimension = new HashMap<>();
    for (String metricOrExpressionName : metricOrExpressionNames) {
        Map<String, Double> baselineTotalMap = new HashMap<>();
        Map<String, Double> currentTotalMap = new HashMap<>();
        baselineTotalPerMetricAndDimension.put(metricOrExpressionName, baselineTotalMap);
        currentTotalPerMetricAndDimension.put(metricOrExpressionName, currentTotalMap);
        for (String dimension : groupByDimensions) {
            baselineTotalMap.put(dimension, 0d);
            currentTotalMap.put(dimension, 0d);
        }
    }
    List<Future<TimeOnTimeComparisonResponse>> timeOnTimeComparisonResponsesFutures = getTimeOnTimeComparisonResponses(groupByDimensions, comparisonRequest, handler);
    for (int groupByDimensionId = 0; groupByDimensionId < groupByDimensions.size(); groupByDimensionId++) {
        String groupByDimension = groupByDimensions.get(groupByDimensionId);
        TimeOnTimeComparisonResponse response = timeOnTimeComparisonResponsesFutures.get(groupByDimensionId).get();
        int numRows = response.getNumRows();
        for (int i = 0; i < numRows; i++) {
            Row row = response.getRow(i);
            String dimensionValue = row.getDimensionValue();
            Map<String, Metric> metricMap = new HashMap<>();
            for (Metric metric : row.getMetrics()) {
                metricMap.put(metric.getMetricName(), metric);
            }
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                // update the baselineTotal and current total
                Map<String, Double> baselineTotalMap = baselineTotalPerMetricAndDimension.get(metricName);
                Map<String, Double> currentTotalMap = currentTotalPerMetricAndDimension.get(metricName);
                baselineTotalMap.put(groupByDimension, baselineTotalMap.get(groupByDimension) + metric.getBaselineValue());
                currentTotalMap.put(groupByDimension, currentTotalMap.get(groupByDimension) + metric.getCurrentValue());
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                String dataKey = metricName + "." + groupByDimension;
                HeatMap.Builder heatMapBuilder = data.get(dataKey);
                if (heatMapBuilder == null) {
                    heatMapBuilder = new HeatMap.Builder(groupByDimension);
                    data.put(dataKey, heatMapBuilder);
                }
                MetricDataset metricDataset = new MetricDataset(metricName, comparisonRequest.getCollectionName());
                MetricConfigDTO metricConfig = CACHE_REGISTRY.getMetricConfigCache().get(metricDataset);
                if (StringUtils.isNotBlank(metricConfig.getCellSizeExpression())) {
                    String metricExpression = metricExpressions.get(metricName);
                    String[] tokens = metricExpression.split(RATIO_SEPARATOR);
                    String numerator = tokens[0];
                    String denominator = tokens[1];
                    Metric numeratorMetric = metricMap.get(numerator);
                    Metric denominatorMetric = metricMap.get(denominator);
                    Double numeratorBaseline = numeratorMetric == null ? 0 : numeratorMetric.getBaselineValue();
                    Double numeratorCurrent = numeratorMetric == null ? 0 : numeratorMetric.getCurrentValue();
                    Double denominatorBaseline = denominatorMetric == null ? 0 : denominatorMetric.getBaselineValue();
                    Double denominatorCurrent = denominatorMetric == null ? 0 : denominatorMetric.getCurrentValue();
                    Map<String, Double> context = new HashMap<>();
                    context.put(numerator, numeratorCurrent);
                    context.put(denominator, denominatorCurrent);
                    String cellSizeExpression = metricConfig.getCellSizeExpression();
                    Double cellSize = MetricExpression.evaluateExpression(cellSizeExpression, context);
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue(), cellSize, cellSizeExpression, numeratorBaseline, denominatorBaseline, numeratorCurrent, denominatorCurrent);
                } else {
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue());
                }
            }
        }
    }
    ResponseSchema schema = new ResponseSchema();
    String[] columns = HeatMapCell.columns();
    for (int i = 0; i < columns.length; i++) {
        String column = columns[i];
        schema.add(column, i);
    }
    Info summary = new Info();
    Map<String, GenericResponse> heatMapViewResponseData = new HashMap<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        Double baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        Double currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        // check if its derived
        if (metricFunctions.size() > 1) {
            Map<String, Double> baselineContext = new HashMap<>();
            Map<String, Double> currentContext = new HashMap<>();
            for (String metricOrExpression : metricOrExpressionNames) {
                baselineContext.put(metricOrExpression, baselineTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
                currentContext.put(metricOrExpression, currentTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
            }
            baselineTotal = MetricExpression.evaluateExpression(expression, baselineContext);
            currentTotal = MetricExpression.evaluateExpression(expression, currentContext);
        } else {
            baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
            currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        }
        summary.addSimpleField("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
        summary.addSimpleField("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
        summary.addSimpleField("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
        summary.addSimpleField("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
        summary.addSimpleField("baselineTotal", HeatMapCell.format(baselineTotal));
        summary.addSimpleField("currentTotal", HeatMapCell.format(currentTotal));
        summary.addSimpleField("deltaChange", HeatMapCell.format(currentTotal - baselineTotal));
        summary.addSimpleField("deltaPercentage", HeatMapCell.format((currentTotal - baselineTotal) * 100.0 / baselineTotal));
    }
    for (Entry<String, HeatMap.Builder> entry : data.entrySet()) {
        String dataKey = entry.getKey();
        GenericResponse heatMapResponse = new GenericResponse();
        List<String[]> heatMapResponseData = new ArrayList<>();
        HeatMap.Builder builder = entry.getValue();
        HeatMap heatMap = builder.build();
        for (HeatMapCell cell : heatMap.heatMapCells) {
            String[] newRowData = cell.toArray();
            heatMapResponseData.add(newRowData);
        }
        heatMapResponse.setSchema(schema);
        heatMapResponse.setResponseData(heatMapResponseData);
        heatMapViewResponseData.put(dataKey, heatMapResponse);
    }
    HeatMapViewResponse heatMapViewResponse = new HeatMapViewResponse();
    heatMapViewResponse.setMetrics(expressionNames);
    heatMapViewResponse.setDimensions(groupByDimensions);
    heatMapViewResponse.setData(heatMapViewResponseData);
    heatMapViewResponse.setMetricExpression(metricExpressions);
    heatMapViewResponse.setSummary(summary);
    return heatMapViewResponse;
}

Example 4 with TimeOnTimeComparisonRequest

use of com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest in project pinot by linkedin.

the class ContributorViewHandler method process.

@Override
public ContributorViewResponse process(ContributorViewRequest request) throws Exception {
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    TimeOnTimeComparisonResponse response = handler.handle(comparisonRequest);
    List<String> metricNames = new ArrayList<>(response.getMetrics());
    List<String> expressionNames = new ArrayList<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
    }
    List<String> dimensions = new ArrayList<>(response.getDimensions());
    List<TimeBucket> timeBuckets = getTimeBuckets(response);
    Map<String, SortedSet<Row>> rows = getRowsSortedByTime(response);
    ContributorViewResponse contributorViewResponse = new ContributorViewResponse();
    contributorViewResponse.setMetrics(expressionNames);
    contributorViewResponse.setDimensions(dimensions);
    contributorViewResponse.setTimeBuckets(timeBuckets);
    GenericResponse genericResponse = new GenericResponse();
    Map<String, Double[]> runningTotalMap = new HashMap<>();
    // one view per <metric,dimensionName> combination
    Map<String, ContributionViewTableBuilder> contributionViewTableMap = new LinkedHashMap<>();
    Map<String, List<String>> dimensionValuesMap = new HashMap<>();
    for (Map.Entry<String, SortedSet<Row>> entry : rows.entrySet()) {
        for (Row row : entry.getValue()) {
            String dimensionName = row.getDimensionName();
            String dimensionValue = row.getDimensionValue();
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                Double baselineValue = metric.getBaselineValue();
                Double currentValue = metric.getCurrentValue();
                Double cumulativeBaselineValue;
                Double cumulativeCurrentValue;
                String metricDimensionNameString = metricName + "." + dimensionName;
                ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
                if (contributionViewTable == null) {
                    contributionViewTable = new ContributionViewTableBuilder(metricName, dimensionName);
                    contributionViewTableMap.put(metricDimensionNameString, contributionViewTable);
                }
                String rowKey = metricName + "." + dimensionName + "." + dimensionValue;
                if (runningTotalMap.containsKey(rowKey)) {
                    Double[] totalValues = runningTotalMap.get(rowKey);
                    cumulativeBaselineValue = totalValues[0] + baselineValue;
                    cumulativeCurrentValue = totalValues[1] + currentValue;
                } else {
                    cumulativeBaselineValue = baselineValue;
                    cumulativeCurrentValue = currentValue;
                }
                TimeBucket timeBucket = TimeBucket.fromRow(row);
                contributionViewTable.addEntry(dimensionValue, timeBucket, baselineValue, currentValue, cumulativeBaselineValue, cumulativeCurrentValue);
                List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
                if (dimensionValues == null) {
                    dimensionValues = new ArrayList<>();
                    dimensionValuesMap.put(dimensionName, dimensionValues);
                }
                if (!dimensionValues.contains(dimensionValue)) {
                    dimensionValues.add(dimensionValue);
                }
                Double[] runningTotalPerMetric = new Double[] { cumulativeBaselineValue, cumulativeCurrentValue };
                runningTotalMap.put(rowKey, runningTotalPerMetric);
            }
        }
    }
    Map<String, List<Integer>> keyToRowIdListMapping = new TreeMap<>();
    List<String[]> rowData = new ArrayList<>();
    // for each metric, dimension pair compute the total value for each dimension. This will be used
    // to sort the dimension values
    Map<String, Map<String, Map<String, Double>>> baselineTotalMapPerDimensionValue = new HashMap<>();
    Map<String, Map<String, Map<String, Double>>> currentTotalMapPerDimensionValue = new HashMap<>();
    for (String metricDimensionNameString : contributionViewTableMap.keySet()) {
        ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
        ContributionViewTable table = contributionViewTable.build();
        List<ContributionCell> cells = table.getCells();
        for (ContributionCell cell : cells) {
            String metricName = table.getMetricName();
            String dimName = table.getDimensionName();
            String dimValue = cell.getDimensionValue();
            String key = metricName + "|" + dimName + "|" + dimValue;
            List<Integer> rowIdList = keyToRowIdListMapping.get(key);
            if (rowIdList == null) {
                rowIdList = new ArrayList<>();
                keyToRowIdListMapping.put(key, rowIdList);
            }
            rowIdList.add(rowData.size());
            rowData.add(cell.toArray());
            // update baseline
            updateTotalForDimensionValue(baselineTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getBaselineValue());
            // update current
            updateTotalForDimensionValue(currentTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getCurrentValue());
        }
    }
    genericResponse.setResponseData(rowData);
    genericResponse.setSchema(new ResponseSchema(ContributionCell.columns()));
    genericResponse.setKeyToRowIdMapping(keyToRowIdListMapping);
    Info summary = new Info();
    genericResponse.setSummary(summary);
    for (String dimensionName : dimensionValuesMap.keySet()) {
        List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
        sort(expressionNames, dimensionName, dimensionValues, baselineTotalMapPerDimensionValue, currentTotalMapPerDimensionValue);
    }
    contributorViewResponse.setDimensionValuesMap(dimensionValuesMap);
    contributorViewResponse.setResponseData(genericResponse);
    contributorViewResponse.setCurrentTotalMapPerDimensionValue(currentTotalMapPerDimensionValue);
    contributorViewResponse.setBaselineTotalMapPerDimensionValue(baselineTotalMapPerDimensionValue);
    return contributorViewResponse;
}

Example 5 with TimeOnTimeComparisonRequest

use of com.linkedin.thirdeye.client.comparison.TimeOnTimeComparisonRequest in project pinot by linkedin.

the class TabularViewHandler method generateTimeOnTimeComparisonRequest.

private TimeOnTimeComparisonRequest generateTimeOnTimeComparisonRequest(TabularViewRequest request) throws Exception {
    TimeOnTimeComparisonRequest comparisonRequest = new TimeOnTimeComparisonRequest();
    String collection = request.getCollection();
    DateTime baselineStart = request.getBaselineStart();
    DateTime baselineEnd = request.getBaselineEnd();
    DateTime currentStart = request.getCurrentStart();
    DateTime currentEnd = request.getCurrentEnd();
    DatasetConfigDTO datasetConfig = CACHE_REGISTRY.getDatasetConfigCache().get(collection);
    TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
    if (!request.getTimeGranularity().getUnit().equals(TimeUnit.DAYS) || !StringUtils.isBlank(timespec.getFormat())) {
        comparisonRequest.setEndDateInclusive(true);
    }
    Multimap<String, String> filters = request.getFilters();
    List<MetricExpression> metricExpressions = request.getMetricExpressions();
    comparisonRequest.setCollectionName(collection);
    comparisonRequest.setBaselineStart(baselineStart);
    comparisonRequest.setBaselineEnd(baselineEnd);
    comparisonRequest.setCurrentStart(currentStart);
    comparisonRequest.setCurrentEnd(currentEnd);
    comparisonRequest.setFilterSet(filters);
    comparisonRequest.setMetricExpressions(metricExpressions);
    comparisonRequest.setAggregationTimeGranularity(request.getTimeGranularity());
    return comparisonRequest;
}