Examples with ResponseSchema com.linkedin.thirdeye.dashboard.views.GenericResponse.ResponseSchema used on opensource projects

Example 1 with ResponseSchema

use of com.linkedin.thirdeye.dashboard.views.GenericResponse.ResponseSchema in project pinot by linkedin.

the class HeatMapViewHandler method process.

@Override
public HeatMapViewResponse process(HeatMapViewRequest request) throws Exception {
    // query 1 for everything from baseline start to baseline end
    // query 2 for everything from current start to current end
    // for each dimension group by top 100
    // query 1 for everything from baseline start to baseline end
    // query for everything from current start to current end
    List<String> expressionNames = new ArrayList<>();
    Map<String, String> metricExpressions = new HashMap<>();
    Set<String> metricOrExpressionNames = new HashSet<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
        metricExpressions.put(expression.getExpressionName(), expression.getExpression());
        metricOrExpressionNames.add(expression.getExpressionName());
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        for (MetricFunction function : metricFunctions) {
            metricOrExpressionNames.add(function.getMetricName());
        }
    }
    Map<String, HeatMap.Builder> data = new HashMap<>();
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    List<String> groupByDimensions = comparisonRequest.getGroupByDimensions();
    final TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    // we are tracking per dimension, to validate that its the same for each dimension
    Map<String, Map<String, Double>> baselineTotalPerMetricAndDimension = new HashMap<>();
    Map<String, Map<String, Double>> currentTotalPerMetricAndDimension = new HashMap<>();
    for (String metricOrExpressionName : metricOrExpressionNames) {
        Map<String, Double> baselineTotalMap = new HashMap<>();
        Map<String, Double> currentTotalMap = new HashMap<>();
        baselineTotalPerMetricAndDimension.put(metricOrExpressionName, baselineTotalMap);
        currentTotalPerMetricAndDimension.put(metricOrExpressionName, currentTotalMap);
        for (String dimension : groupByDimensions) {
            baselineTotalMap.put(dimension, 0d);
            currentTotalMap.put(dimension, 0d);
        }
    }
    List<Future<TimeOnTimeComparisonResponse>> timeOnTimeComparisonResponsesFutures = getTimeOnTimeComparisonResponses(groupByDimensions, comparisonRequest, handler);
    for (int groupByDimensionId = 0; groupByDimensionId < groupByDimensions.size(); groupByDimensionId++) {
        String groupByDimension = groupByDimensions.get(groupByDimensionId);
        TimeOnTimeComparisonResponse response = timeOnTimeComparisonResponsesFutures.get(groupByDimensionId).get();
        int numRows = response.getNumRows();
        for (int i = 0; i < numRows; i++) {
            Row row = response.getRow(i);
            String dimensionValue = row.getDimensionValue();
            Map<String, Metric> metricMap = new HashMap<>();
            for (Metric metric : row.getMetrics()) {
                metricMap.put(metric.getMetricName(), metric);
            }
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                // update the baselineTotal and current total
                Map<String, Double> baselineTotalMap = baselineTotalPerMetricAndDimension.get(metricName);
                Map<String, Double> currentTotalMap = currentTotalPerMetricAndDimension.get(metricName);
                baselineTotalMap.put(groupByDimension, baselineTotalMap.get(groupByDimension) + metric.getBaselineValue());
                currentTotalMap.put(groupByDimension, currentTotalMap.get(groupByDimension) + metric.getCurrentValue());
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                String dataKey = metricName + "." + groupByDimension;
                HeatMap.Builder heatMapBuilder = data.get(dataKey);
                if (heatMapBuilder == null) {
                    heatMapBuilder = new HeatMap.Builder(groupByDimension);
                    data.put(dataKey, heatMapBuilder);
                }
                MetricDataset metricDataset = new MetricDataset(metricName, comparisonRequest.getCollectionName());
                MetricConfigDTO metricConfig = CACHE_REGISTRY.getMetricConfigCache().get(metricDataset);
                if (StringUtils.isNotBlank(metricConfig.getCellSizeExpression())) {
                    String metricExpression = metricExpressions.get(metricName);
                    String[] tokens = metricExpression.split(RATIO_SEPARATOR);
                    String numerator = tokens[0];
                    String denominator = tokens[1];
                    Metric numeratorMetric = metricMap.get(numerator);
                    Metric denominatorMetric = metricMap.get(denominator);
                    Double numeratorBaseline = numeratorMetric == null ? 0 : numeratorMetric.getBaselineValue();
                    Double numeratorCurrent = numeratorMetric == null ? 0 : numeratorMetric.getCurrentValue();
                    Double denominatorBaseline = denominatorMetric == null ? 0 : denominatorMetric.getBaselineValue();
                    Double denominatorCurrent = denominatorMetric == null ? 0 : denominatorMetric.getCurrentValue();
                    Map<String, Double> context = new HashMap<>();
                    context.put(numerator, numeratorCurrent);
                    context.put(denominator, denominatorCurrent);
                    String cellSizeExpression = metricConfig.getCellSizeExpression();
                    Double cellSize = MetricExpression.evaluateExpression(cellSizeExpression, context);
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue(), cellSize, cellSizeExpression, numeratorBaseline, denominatorBaseline, numeratorCurrent, denominatorCurrent);
                } else {
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue());
                }
            }
        }
    }
    ResponseSchema schema = new ResponseSchema();
    String[] columns = HeatMapCell.columns();
    for (int i = 0; i < columns.length; i++) {
        String column = columns[i];
        schema.add(column, i);
    }
    Info summary = new Info();
    Map<String, GenericResponse> heatMapViewResponseData = new HashMap<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        Double baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        Double currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        // check if its derived
        if (metricFunctions.size() > 1) {
            Map<String, Double> baselineContext = new HashMap<>();
            Map<String, Double> currentContext = new HashMap<>();
            for (String metricOrExpression : metricOrExpressionNames) {
                baselineContext.put(metricOrExpression, baselineTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
                currentContext.put(metricOrExpression, currentTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
            }
            baselineTotal = MetricExpression.evaluateExpression(expression, baselineContext);
            currentTotal = MetricExpression.evaluateExpression(expression, currentContext);
        } else {
            baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
            currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        }
        summary.addSimpleField("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
        summary.addSimpleField("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
        summary.addSimpleField("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
        summary.addSimpleField("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
        summary.addSimpleField("baselineTotal", HeatMapCell.format(baselineTotal));
        summary.addSimpleField("currentTotal", HeatMapCell.format(currentTotal));
        summary.addSimpleField("deltaChange", HeatMapCell.format(currentTotal - baselineTotal));
        summary.addSimpleField("deltaPercentage", HeatMapCell.format((currentTotal - baselineTotal) * 100.0 / baselineTotal));
    }
    for (Entry<String, HeatMap.Builder> entry : data.entrySet()) {
        String dataKey = entry.getKey();
        GenericResponse heatMapResponse = new GenericResponse();
        List<String[]> heatMapResponseData = new ArrayList<>();
        HeatMap.Builder builder = entry.getValue();
        HeatMap heatMap = builder.build();
        for (HeatMapCell cell : heatMap.heatMapCells) {
            String[] newRowData = cell.toArray();
            heatMapResponseData.add(newRowData);
        }
        heatMapResponse.setSchema(schema);
        heatMapResponse.setResponseData(heatMapResponseData);
        heatMapViewResponseData.put(dataKey, heatMapResponse);
    }
    HeatMapViewResponse heatMapViewResponse = new HeatMapViewResponse();
    heatMapViewResponse.setMetrics(expressionNames);
    heatMapViewResponse.setDimensions(groupByDimensions);
    heatMapViewResponse.setData(heatMapViewResponseData);
    heatMapViewResponse.setMetricExpression(metricExpressions);
    heatMapViewResponse.setSummary(summary);
    return heatMapViewResponse;
}

Example 2 with ResponseSchema

use of com.linkedin.thirdeye.dashboard.views.GenericResponse.ResponseSchema in project pinot by linkedin.

the class ContributorViewHandler method process.

@Override
public ContributorViewResponse process(ContributorViewRequest request) throws Exception {
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    TimeOnTimeComparisonResponse response = handler.handle(comparisonRequest);
    List<String> metricNames = new ArrayList<>(response.getMetrics());
    List<String> expressionNames = new ArrayList<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
    }
    List<String> dimensions = new ArrayList<>(response.getDimensions());
    List<TimeBucket> timeBuckets = getTimeBuckets(response);
    Map<String, SortedSet<Row>> rows = getRowsSortedByTime(response);
    ContributorViewResponse contributorViewResponse = new ContributorViewResponse();
    contributorViewResponse.setMetrics(expressionNames);
    contributorViewResponse.setDimensions(dimensions);
    contributorViewResponse.setTimeBuckets(timeBuckets);
    GenericResponse genericResponse = new GenericResponse();
    Map<String, Double[]> runningTotalMap = new HashMap<>();
    // one view per <metric,dimensionName> combination
    Map<String, ContributionViewTableBuilder> contributionViewTableMap = new LinkedHashMap<>();
    Map<String, List<String>> dimensionValuesMap = new HashMap<>();
    for (Map.Entry<String, SortedSet<Row>> entry : rows.entrySet()) {
        for (Row row : entry.getValue()) {
            String dimensionName = row.getDimensionName();
            String dimensionValue = row.getDimensionValue();
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                Double baselineValue = metric.getBaselineValue();
                Double currentValue = metric.getCurrentValue();
                Double cumulativeBaselineValue;
                Double cumulativeCurrentValue;
                String metricDimensionNameString = metricName + "." + dimensionName;
                ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
                if (contributionViewTable == null) {
                    contributionViewTable = new ContributionViewTableBuilder(metricName, dimensionName);
                    contributionViewTableMap.put(metricDimensionNameString, contributionViewTable);
                }
                String rowKey = metricName + "." + dimensionName + "." + dimensionValue;
                if (runningTotalMap.containsKey(rowKey)) {
                    Double[] totalValues = runningTotalMap.get(rowKey);
                    cumulativeBaselineValue = totalValues[0] + baselineValue;
                    cumulativeCurrentValue = totalValues[1] + currentValue;
                } else {
                    cumulativeBaselineValue = baselineValue;
                    cumulativeCurrentValue = currentValue;
                }
                TimeBucket timeBucket = TimeBucket.fromRow(row);
                contributionViewTable.addEntry(dimensionValue, timeBucket, baselineValue, currentValue, cumulativeBaselineValue, cumulativeCurrentValue);
                List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
                if (dimensionValues == null) {
                    dimensionValues = new ArrayList<>();
                    dimensionValuesMap.put(dimensionName, dimensionValues);
                }
                if (!dimensionValues.contains(dimensionValue)) {
                    dimensionValues.add(dimensionValue);
                }
                Double[] runningTotalPerMetric = new Double[] { cumulativeBaselineValue, cumulativeCurrentValue };
                runningTotalMap.put(rowKey, runningTotalPerMetric);
            }
        }
    }
    Map<String, List<Integer>> keyToRowIdListMapping = new TreeMap<>();
    List<String[]> rowData = new ArrayList<>();
    // for each metric, dimension pair compute the total value for each dimension. This will be used
    // to sort the dimension values
    Map<String, Map<String, Map<String, Double>>> baselineTotalMapPerDimensionValue = new HashMap<>();
    Map<String, Map<String, Map<String, Double>>> currentTotalMapPerDimensionValue = new HashMap<>();
    for (String metricDimensionNameString : contributionViewTableMap.keySet()) {
        ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
        ContributionViewTable table = contributionViewTable.build();
        List<ContributionCell> cells = table.getCells();
        for (ContributionCell cell : cells) {
            String metricName = table.getMetricName();
            String dimName = table.getDimensionName();
            String dimValue = cell.getDimensionValue();
            String key = metricName + "|" + dimName + "|" + dimValue;
            List<Integer> rowIdList = keyToRowIdListMapping.get(key);
            if (rowIdList == null) {
                rowIdList = new ArrayList<>();
                keyToRowIdListMapping.put(key, rowIdList);
            }
            rowIdList.add(rowData.size());
            rowData.add(cell.toArray());
            // update baseline
            updateTotalForDimensionValue(baselineTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getBaselineValue());
            // update current
            updateTotalForDimensionValue(currentTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getCurrentValue());
        }
    }
    genericResponse.setResponseData(rowData);
    genericResponse.setSchema(new ResponseSchema(ContributionCell.columns()));
    genericResponse.setKeyToRowIdMapping(keyToRowIdListMapping);
    Info summary = new Info();
    genericResponse.setSummary(summary);
    for (String dimensionName : dimensionValuesMap.keySet()) {
        List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
        sort(expressionNames, dimensionName, dimensionValues, baselineTotalMapPerDimensionValue, currentTotalMapPerDimensionValue);
    }
    contributorViewResponse.setDimensionValuesMap(dimensionValuesMap);
    contributorViewResponse.setResponseData(genericResponse);
    contributorViewResponse.setCurrentTotalMapPerDimensionValue(currentTotalMapPerDimensionValue);
    contributorViewResponse.setBaselineTotalMapPerDimensionValue(baselineTotalMapPerDimensionValue);
    return contributorViewResponse;
}

Example 3 with ResponseSchema

use of com.linkedin.thirdeye.dashboard.views.GenericResponse.ResponseSchema in project pinot by linkedin.

the class TabularViewHandler method addColumnData.

public void addColumnData(Map<String, GenericResponse> data, String metric, String[] columnData, String[] columns) {
    GenericResponse rowData;
    if (data.containsKey(metric)) {
        rowData = data.get(metric);
        rowData.getResponseData().add(columnData);
    } else {
        ResponseSchema schema = new ResponseSchema();
        for (int i = 0; i < columns.length; i++) {
            String column = columns[i];
            schema.add(column, i);
        }
        rowData = new GenericResponse();
        List<String[]> genericResponseData = new ArrayList<>();
        genericResponseData.add(columnData);
        rowData.setResponseData(genericResponseData);
        rowData.setSchema(schema);
        data.put(metric, rowData);
    }
}

Example 4 with ResponseSchema

use of com.linkedin.thirdeye.dashboard.views.GenericResponse.ResponseSchema in project pinot by linkedin.

the class TabularViewHandler method process.

@Override
public TabularViewResponse process(TabularViewRequest request) throws Exception {
    // query 1 for everything from baseline start to baseline end
    // query 2 for everything from current start to current end
    // for each dimension group by top 100
    // query 1 for everything from baseline start to baseline end
    // query for everything from current start to current end
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    TimeOnTimeComparisonResponse response = handler.handle(comparisonRequest);
    List<TimeBucket> timeBuckets = getTimeBuckets(response);
    List<Row> rows = getRowsSortedByTime(response);
    TabularViewResponse tabularViewResponse = new TabularViewResponse();
    Map<String, String> summary = new HashMap<>();
    summary.put("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
    summary.put("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
    summary.put("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
    summary.put("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
    tabularViewResponse.setSummary(summary);
    List<String> expressionNames = new ArrayList<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
    }
    tabularViewResponse.setMetrics(expressionNames);
    tabularViewResponse.setTimeBuckets(timeBuckets);
    String[] columns = new String[] { "baselineValue", "currentValue", "ratio", "cumulativeBaselineValue", "cumulativeCurrentValue", "cumulativeRatio" };
    // maintain same order in response
    Map<String, GenericResponse> data = new LinkedHashMap<>();
    for (String metric : expressionNames) {
        ResponseSchema schema = new ResponseSchema();
        for (int i = 0; i < columns.length; i++) {
            String column = columns[i];
            schema.add(column, i);
        }
        GenericResponse rowData = new GenericResponse();
        rowData.setSchema(schema);
        List<String[]> genericResponseData = new ArrayList<>();
        rowData.setResponseData(genericResponseData);
        data.put(metric, rowData);
    }
    tabularViewResponse.setData(data);
    Map<String, Double[]> runningTotalMap = new HashMap<>();
    for (Row row : rows) {
        for (Metric metric : row.getMetrics()) {
            String metricName = metric.getMetricName();
            if (!expressionNames.contains(metricName)) {
                continue;
            }
            Double baselineValue = metric.getBaselineValue();
            Double currentValue = metric.getCurrentValue();
            String baselineValueStr = HeatMapCell.format(baselineValue);
            String currentValueStr = HeatMapCell.format(currentValue);
            double ratio = ((currentValue - baselineValue) * 100) / baselineValue;
            if (Double.isNaN(ratio)) {
                ratio = 0;
            } else if (Double.isInfinite(ratio)) {
                ratio = 100;
            }
            String ratioStr = HeatMapCell.format(ratio);
            Double cumulativeBaselineValue;
            Double cumulativeCurrentValue;
            if (runningTotalMap.containsKey(metricName)) {
                Double[] totalValues = runningTotalMap.get(metricName);
                cumulativeBaselineValue = totalValues[0] + baselineValue;
                cumulativeCurrentValue = totalValues[1] + currentValue;
            } else {
                cumulativeBaselineValue = baselineValue;
                cumulativeCurrentValue = currentValue;
            }
            Double[] runningTotalPerMetric = new Double[] { cumulativeBaselineValue, cumulativeCurrentValue };
            runningTotalMap.put(metricName, runningTotalPerMetric);
            String cumulativeBaselineValueStr = HeatMapCell.format(cumulativeBaselineValue);
            String cumulativeCurrentValueStr = HeatMapCell.format(cumulativeCurrentValue);
            double cumulativeRatio = ((cumulativeCurrentValue - cumulativeBaselineValue) * 100) / cumulativeBaselineValue;
            if (Double.isNaN(ratio)) {
                cumulativeRatio = 0;
            } else if (Double.isInfinite(cumulativeRatio)) {
                cumulativeRatio = 100;
            }
            String cumulativeRatioStr = HeatMapCell.format(cumulativeRatio);
            String[] columnData = { baselineValueStr, currentValueStr, ratioStr, cumulativeBaselineValueStr, cumulativeCurrentValueStr, cumulativeRatioStr };
            GenericResponse rowData = data.get(metric.getMetricName());
            rowData.getResponseData().add(columnData);
        }
    }
    return tabularViewResponse;
}