use of com.linkedin.thirdeye.dashboard.views.GenericResponse in project pinot by linkedin.
the class HeatMapViewHandler method process.
@Override
public HeatMapViewResponse process(HeatMapViewRequest request) throws Exception {
// query 1 for everything from baseline start to baseline end
// query 2 for everything from current start to current end
// for each dimension group by top 100
// query 1 for everything from baseline start to baseline end
// query for everything from current start to current end
List<String> expressionNames = new ArrayList<>();
Map<String, String> metricExpressions = new HashMap<>();
Set<String> metricOrExpressionNames = new HashSet<>();
for (MetricExpression expression : request.getMetricExpressions()) {
expressionNames.add(expression.getExpressionName());
metricExpressions.put(expression.getExpressionName(), expression.getExpression());
metricOrExpressionNames.add(expression.getExpressionName());
List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
for (MetricFunction function : metricFunctions) {
metricOrExpressionNames.add(function.getMetricName());
}
}
Map<String, HeatMap.Builder> data = new HashMap<>();
TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
List<String> groupByDimensions = comparisonRequest.getGroupByDimensions();
final TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
// we are tracking per dimension, to validate that its the same for each dimension
Map<String, Map<String, Double>> baselineTotalPerMetricAndDimension = new HashMap<>();
Map<String, Map<String, Double>> currentTotalPerMetricAndDimension = new HashMap<>();
for (String metricOrExpressionName : metricOrExpressionNames) {
Map<String, Double> baselineTotalMap = new HashMap<>();
Map<String, Double> currentTotalMap = new HashMap<>();
baselineTotalPerMetricAndDimension.put(metricOrExpressionName, baselineTotalMap);
currentTotalPerMetricAndDimension.put(metricOrExpressionName, currentTotalMap);
for (String dimension : groupByDimensions) {
baselineTotalMap.put(dimension, 0d);
currentTotalMap.put(dimension, 0d);
}
}
List<Future<TimeOnTimeComparisonResponse>> timeOnTimeComparisonResponsesFutures = getTimeOnTimeComparisonResponses(groupByDimensions, comparisonRequest, handler);
for (int groupByDimensionId = 0; groupByDimensionId < groupByDimensions.size(); groupByDimensionId++) {
String groupByDimension = groupByDimensions.get(groupByDimensionId);
TimeOnTimeComparisonResponse response = timeOnTimeComparisonResponsesFutures.get(groupByDimensionId).get();
int numRows = response.getNumRows();
for (int i = 0; i < numRows; i++) {
Row row = response.getRow(i);
String dimensionValue = row.getDimensionValue();
Map<String, Metric> metricMap = new HashMap<>();
for (Metric metric : row.getMetrics()) {
metricMap.put(metric.getMetricName(), metric);
}
for (Metric metric : row.getMetrics()) {
String metricName = metric.getMetricName();
// update the baselineTotal and current total
Map<String, Double> baselineTotalMap = baselineTotalPerMetricAndDimension.get(metricName);
Map<String, Double> currentTotalMap = currentTotalPerMetricAndDimension.get(metricName);
baselineTotalMap.put(groupByDimension, baselineTotalMap.get(groupByDimension) + metric.getBaselineValue());
currentTotalMap.put(groupByDimension, currentTotalMap.get(groupByDimension) + metric.getCurrentValue());
if (!expressionNames.contains(metricName)) {
continue;
}
String dataKey = metricName + "." + groupByDimension;
HeatMap.Builder heatMapBuilder = data.get(dataKey);
if (heatMapBuilder == null) {
heatMapBuilder = new HeatMap.Builder(groupByDimension);
data.put(dataKey, heatMapBuilder);
}
MetricDataset metricDataset = new MetricDataset(metricName, comparisonRequest.getCollectionName());
MetricConfigDTO metricConfig = CACHE_REGISTRY.getMetricConfigCache().get(metricDataset);
if (StringUtils.isNotBlank(metricConfig.getCellSizeExpression())) {
String metricExpression = metricExpressions.get(metricName);
String[] tokens = metricExpression.split(RATIO_SEPARATOR);
String numerator = tokens[0];
String denominator = tokens[1];
Metric numeratorMetric = metricMap.get(numerator);
Metric denominatorMetric = metricMap.get(denominator);
Double numeratorBaseline = numeratorMetric == null ? 0 : numeratorMetric.getBaselineValue();
Double numeratorCurrent = numeratorMetric == null ? 0 : numeratorMetric.getCurrentValue();
Double denominatorBaseline = denominatorMetric == null ? 0 : denominatorMetric.getBaselineValue();
Double denominatorCurrent = denominatorMetric == null ? 0 : denominatorMetric.getCurrentValue();
Map<String, Double> context = new HashMap<>();
context.put(numerator, numeratorCurrent);
context.put(denominator, denominatorCurrent);
String cellSizeExpression = metricConfig.getCellSizeExpression();
Double cellSize = MetricExpression.evaluateExpression(cellSizeExpression, context);
heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue(), cellSize, cellSizeExpression, numeratorBaseline, denominatorBaseline, numeratorCurrent, denominatorCurrent);
} else {
heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue());
}
}
}
}
ResponseSchema schema = new ResponseSchema();
String[] columns = HeatMapCell.columns();
for (int i = 0; i < columns.length; i++) {
String column = columns[i];
schema.add(column, i);
}
Info summary = new Info();
Map<String, GenericResponse> heatMapViewResponseData = new HashMap<>();
for (MetricExpression expression : request.getMetricExpressions()) {
List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
Double baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
Double currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
// check if its derived
if (metricFunctions.size() > 1) {
Map<String, Double> baselineContext = new HashMap<>();
Map<String, Double> currentContext = new HashMap<>();
for (String metricOrExpression : metricOrExpressionNames) {
baselineContext.put(metricOrExpression, baselineTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
currentContext.put(metricOrExpression, currentTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
}
baselineTotal = MetricExpression.evaluateExpression(expression, baselineContext);
currentTotal = MetricExpression.evaluateExpression(expression, currentContext);
} else {
baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
}
summary.addSimpleField("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
summary.addSimpleField("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
summary.addSimpleField("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
summary.addSimpleField("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
summary.addSimpleField("baselineTotal", HeatMapCell.format(baselineTotal));
summary.addSimpleField("currentTotal", HeatMapCell.format(currentTotal));
summary.addSimpleField("deltaChange", HeatMapCell.format(currentTotal - baselineTotal));
summary.addSimpleField("deltaPercentage", HeatMapCell.format((currentTotal - baselineTotal) * 100.0 / baselineTotal));
}
for (Entry<String, HeatMap.Builder> entry : data.entrySet()) {
String dataKey = entry.getKey();
GenericResponse heatMapResponse = new GenericResponse();
List<String[]> heatMapResponseData = new ArrayList<>();
HeatMap.Builder builder = entry.getValue();
HeatMap heatMap = builder.build();
for (HeatMapCell cell : heatMap.heatMapCells) {
String[] newRowData = cell.toArray();
heatMapResponseData.add(newRowData);
}
heatMapResponse.setSchema(schema);
heatMapResponse.setResponseData(heatMapResponseData);
heatMapViewResponseData.put(dataKey, heatMapResponse);
}
HeatMapViewResponse heatMapViewResponse = new HeatMapViewResponse();
heatMapViewResponse.setMetrics(expressionNames);
heatMapViewResponse.setDimensions(groupByDimensions);
heatMapViewResponse.setData(heatMapViewResponseData);
heatMapViewResponse.setMetricExpression(metricExpressions);
heatMapViewResponse.setSummary(summary);
return heatMapViewResponse;
}
use of com.linkedin.thirdeye.dashboard.views.GenericResponse in project pinot by linkedin.
the class ContributorViewHandler method process.
@Override
public ContributorViewResponse process(ContributorViewRequest request) throws Exception {
TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
TimeOnTimeComparisonResponse response = handler.handle(comparisonRequest);
List<String> metricNames = new ArrayList<>(response.getMetrics());
List<String> expressionNames = new ArrayList<>();
for (MetricExpression expression : request.getMetricExpressions()) {
expressionNames.add(expression.getExpressionName());
}
List<String> dimensions = new ArrayList<>(response.getDimensions());
List<TimeBucket> timeBuckets = getTimeBuckets(response);
Map<String, SortedSet<Row>> rows = getRowsSortedByTime(response);
ContributorViewResponse contributorViewResponse = new ContributorViewResponse();
contributorViewResponse.setMetrics(expressionNames);
contributorViewResponse.setDimensions(dimensions);
contributorViewResponse.setTimeBuckets(timeBuckets);
GenericResponse genericResponse = new GenericResponse();
Map<String, Double[]> runningTotalMap = new HashMap<>();
// one view per <metric,dimensionName> combination
Map<String, ContributionViewTableBuilder> contributionViewTableMap = new LinkedHashMap<>();
Map<String, List<String>> dimensionValuesMap = new HashMap<>();
for (Map.Entry<String, SortedSet<Row>> entry : rows.entrySet()) {
for (Row row : entry.getValue()) {
String dimensionName = row.getDimensionName();
String dimensionValue = row.getDimensionValue();
for (Metric metric : row.getMetrics()) {
String metricName = metric.getMetricName();
if (!expressionNames.contains(metricName)) {
continue;
}
Double baselineValue = metric.getBaselineValue();
Double currentValue = metric.getCurrentValue();
Double cumulativeBaselineValue;
Double cumulativeCurrentValue;
String metricDimensionNameString = metricName + "." + dimensionName;
ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
if (contributionViewTable == null) {
contributionViewTable = new ContributionViewTableBuilder(metricName, dimensionName);
contributionViewTableMap.put(metricDimensionNameString, contributionViewTable);
}
String rowKey = metricName + "." + dimensionName + "." + dimensionValue;
if (runningTotalMap.containsKey(rowKey)) {
Double[] totalValues = runningTotalMap.get(rowKey);
cumulativeBaselineValue = totalValues[0] + baselineValue;
cumulativeCurrentValue = totalValues[1] + currentValue;
} else {
cumulativeBaselineValue = baselineValue;
cumulativeCurrentValue = currentValue;
}
TimeBucket timeBucket = TimeBucket.fromRow(row);
contributionViewTable.addEntry(dimensionValue, timeBucket, baselineValue, currentValue, cumulativeBaselineValue, cumulativeCurrentValue);
List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
if (dimensionValues == null) {
dimensionValues = new ArrayList<>();
dimensionValuesMap.put(dimensionName, dimensionValues);
}
if (!dimensionValues.contains(dimensionValue)) {
dimensionValues.add(dimensionValue);
}
Double[] runningTotalPerMetric = new Double[] { cumulativeBaselineValue, cumulativeCurrentValue };
runningTotalMap.put(rowKey, runningTotalPerMetric);
}
}
}
Map<String, List<Integer>> keyToRowIdListMapping = new TreeMap<>();
List<String[]> rowData = new ArrayList<>();
// for each metric, dimension pair compute the total value for each dimension. This will be used
// to sort the dimension values
Map<String, Map<String, Map<String, Double>>> baselineTotalMapPerDimensionValue = new HashMap<>();
Map<String, Map<String, Map<String, Double>>> currentTotalMapPerDimensionValue = new HashMap<>();
for (String metricDimensionNameString : contributionViewTableMap.keySet()) {
ContributionViewTableBuilder contributionViewTable = contributionViewTableMap.get(metricDimensionNameString);
ContributionViewTable table = contributionViewTable.build();
List<ContributionCell> cells = table.getCells();
for (ContributionCell cell : cells) {
String metricName = table.getMetricName();
String dimName = table.getDimensionName();
String dimValue = cell.getDimensionValue();
String key = metricName + "|" + dimName + "|" + dimValue;
List<Integer> rowIdList = keyToRowIdListMapping.get(key);
if (rowIdList == null) {
rowIdList = new ArrayList<>();
keyToRowIdListMapping.put(key, rowIdList);
}
rowIdList.add(rowData.size());
rowData.add(cell.toArray());
// update baseline
updateTotalForDimensionValue(baselineTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getBaselineValue());
// update current
updateTotalForDimensionValue(currentTotalMapPerDimensionValue, metricName, dimName, dimValue, cell.getCurrentValue());
}
}
genericResponse.setResponseData(rowData);
genericResponse.setSchema(new ResponseSchema(ContributionCell.columns()));
genericResponse.setKeyToRowIdMapping(keyToRowIdListMapping);
Info summary = new Info();
genericResponse.setSummary(summary);
for (String dimensionName : dimensionValuesMap.keySet()) {
List<String> dimensionValues = dimensionValuesMap.get(dimensionName);
sort(expressionNames, dimensionName, dimensionValues, baselineTotalMapPerDimensionValue, currentTotalMapPerDimensionValue);
}
contributorViewResponse.setDimensionValuesMap(dimensionValuesMap);
contributorViewResponse.setResponseData(genericResponse);
contributorViewResponse.setCurrentTotalMapPerDimensionValue(currentTotalMapPerDimensionValue);
contributorViewResponse.setBaselineTotalMapPerDimensionValue(baselineTotalMapPerDimensionValue);
return contributorViewResponse;
}
use of com.linkedin.thirdeye.dashboard.views.GenericResponse in project pinot by linkedin.
the class HeatMapViewResponse method updateStats.
public void updateStats() {
for (Map.Entry<String, GenericResponse> entry : data.entrySet()) {
GenericResponse genericResponse = entry.getValue();
Map<String, Integer> columnsToIndexMapping = genericResponse.getSchema().getColumnsToIndexMapping();
int i = columnsToIndexMapping.get("baselineValue");
int j = columnsToIndexMapping.get("currentValue");
for (String[] rowData : genericResponse.getResponseData()) {
Double baselineValue = Double.valueOf(rowData[i]);
Double currentValue = Double.valueOf(rowData[j]);
DescriptiveStatistics baselineStats = new DescriptiveStatistics();
DescriptiveStatistics currentStats = new DescriptiveStatistics();
// Add stats
// Update columns
// Update column to index mapping
}
}
}
use of com.linkedin.thirdeye.dashboard.views.GenericResponse in project pinot by linkedin.
the class DataResource method getWowSummary.
@GET
@Path("dashboard/wowsummary")
public WowSummary getWowSummary(@QueryParam("dashboard") String dashboard, @QueryParam("timeRanges") String timeRanges) {
WowSummary wowSummary = new WowSummary();
if (StringUtils.isBlank(dashboard)) {
return wowSummary;
}
List<Long> metricIds = getMetricIdsByDashboard(dashboard);
List<String> timeRangeLabels = Lists.newArrayList(timeRanges.split(","));
// Sort metric's id and metric expression by collections
Multimap<String, Long> datasetToMetrics = ArrayListMultimap.create();
Multimap<String, MetricExpression> datasetToMetricExpressions = ArrayListMultimap.create();
Map<Long, MetricConfigDTO> metricIdToMetricConfig = new HashMap<>();
for (long metricId : metricIds) {
MetricConfigDTO metricConfig = metricConfigDAO.findById(metricId);
metricIdToMetricConfig.put(metricId, metricConfig);
datasetToMetrics.put(metricConfig.getDataset(), metricId);
datasetToMetricExpressions.put(metricConfig.getDataset(), ThirdEyeUtils.getMetricExpressionFromMetricConfig(metricConfig));
}
Multimap<String, MetricSummary> metricAliasToMetricSummariesMap = ArrayListMultimap.create();
// Create query request for each collection
for (String dataset : datasetToMetrics.keySet()) {
TabularViewRequest request = new TabularViewRequest();
request.setCollection(dataset);
request.setMetricExpressions(new ArrayList<>(datasetToMetricExpressions.get(dataset)));
// user and server's timezone, including daylight saving time.
for (String timeRangeLabel : timeRangeLabels) {
DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(dataset);
TimeRange timeRange = getTimeRangeFromLabel(dataset, timeZoneForCollection, timeRangeLabel);
long currentEnd = timeRange.getEnd();
long currentStart = timeRange.getStart();
System.out.println(timeRangeLabel + "Current start end " + new DateTime(currentStart) + " " + new DateTime(currentEnd));
TimeGranularity timeGranularity = new TimeGranularity(1, TimeUnit.HOURS);
request.setBaselineStart(new DateTime(currentStart, timeZoneForCollection).minusDays(7));
request.setBaselineEnd(new DateTime(currentEnd, timeZoneForCollection).minusDays(7));
request.setCurrentStart(new DateTime(currentStart, timeZoneForCollection));
request.setCurrentEnd(new DateTime(currentEnd, timeZoneForCollection));
request.setTimeGranularity(timeGranularity);
TabularViewHandler handler = new TabularViewHandler(queryCache);
try {
TabularViewResponse tabularViewResponse = handler.process(request);
for (String metric : tabularViewResponse.getMetrics()) {
MetricDataset metricDataset = new MetricDataset(metric, dataset);
MetricConfigDTO metricConfig = CACHE_REGISTRY_INSTANCE.getMetricConfigCache().get(metricDataset);
Long metricId = metricConfig.getId();
String metricAlias = metricConfig.getAlias();
GenericResponse response = tabularViewResponse.getData().get(metric);
MetricSummary metricSummary = new MetricSummary();
metricSummary.setMetricId(metricId);
metricSummary.setMetricName(metricConfig.getName());
metricSummary.setMetricAlias(metricAlias);
List<String[]> data = response.getResponseData();
double baselineValue = 0;
double currentValue = 0;
for (String[] responseData : data) {
baselineValue = baselineValue + Double.valueOf(responseData[0]);
currentValue = currentValue + Double.valueOf(responseData[1]);
}
double percentageChange = (currentValue - baselineValue) * 100 / baselineValue;
metricSummary.setBaselineValue(baselineValue);
metricSummary.setCurrentValue(currentValue);
metricSummary.setWowPercentageChange(percentageChange);
metricAliasToMetricSummariesMap.put(metricAlias, metricSummary);
}
} catch (Exception e) {
LOG.error("Exception while processing /data/tabular call", e);
}
}
}
wowSummary.setMetricAliasToMetricSummariesMap(metricAliasToMetricSummariesMap);
return wowSummary;
}
use of com.linkedin.thirdeye.dashboard.views.GenericResponse in project pinot by linkedin.
the class DataResource method getMetricSummary.
/**
* Returns percentage change between current values and baseline values. The values are
* aggregated according to the number of buckets. If the bucket number is 1, then all values
* between the given time ranges are sorted to the corresponding bucket and aggregated.
*
* Note: For current implementation, we assume the number of buckets is always 1.
*/
@GET
@Path("dashboard/metricsummary")
public List<MetricSummary> getMetricSummary(@QueryParam("dashboard") String dashboard, @QueryParam("timeRange") String timeRange) {
List<MetricSummary> metricsSummary = new ArrayList<>();
if (StringUtils.isBlank(dashboard)) {
return metricsSummary;
}
List<Long> metricIds = getMetricIdsByDashboard(dashboard);
// Sort metric's id and metric expression by collections
Multimap<String, Long> datasetToMetrics = ArrayListMultimap.create();
Multimap<String, MetricExpression> datasetToMetricExpressions = ArrayListMultimap.create();
Map<Long, MetricConfigDTO> metricIdToMetricConfig = new HashMap<>();
for (long metricId : metricIds) {
MetricConfigDTO metricConfig = metricConfigDAO.findById(metricId);
metricIdToMetricConfig.put(metricId, metricConfig);
datasetToMetrics.put(metricConfig.getDataset(), metricId);
datasetToMetricExpressions.put(metricConfig.getDataset(), ThirdEyeUtils.getMetricExpressionFromMetricConfig(metricConfig));
}
// Create query request for each collection
for (String dataset : datasetToMetrics.keySet()) {
TabularViewRequest request = new TabularViewRequest();
request.setCollection(dataset);
request.setMetricExpressions(new ArrayList<>(datasetToMetricExpressions.get(dataset)));
// The input start and end time (i.e., currentStart, currentEnd, baselineStart, and
// baselineEnd) are given in millisecond since epoch, which is timezone insensitive. On the
// other hand, the start and end time of the request to be sent to backend database (e.g.,
// Pinot) could be converted to SimpleDateFormat, which is timezone sensitive. Therefore,
// we need to store user's start and end time in DateTime objects with data's timezone
// in order to ensure that the conversion to SimpleDateFormat is always correct regardless
// user and server's timezone, including daylight saving time.
String[] tokens = timeRange.split("_");
TimeGranularity timeGranularity = new TimeGranularity(Integer.valueOf(tokens[0]), TimeUnit.valueOf(tokens[1]));
long currentEnd = Utils.getMaxDataTimeForDataset(dataset);
long currentStart = currentEnd - TimeUnit.MILLISECONDS.convert(Long.valueOf(tokens[0]), TimeUnit.valueOf(tokens[1]));
DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(dataset);
request.setBaselineStart(new DateTime(currentStart, timeZoneForCollection).minusDays(7));
request.setBaselineEnd(new DateTime(currentEnd, timeZoneForCollection).minusDays(7));
request.setCurrentStart(new DateTime(currentStart, timeZoneForCollection));
request.setCurrentEnd(new DateTime(currentEnd, timeZoneForCollection));
request.setTimeGranularity(timeGranularity);
TabularViewHandler handler = new TabularViewHandler(queryCache);
try {
TabularViewResponse tabularViewResponse = handler.process(request);
for (String metric : tabularViewResponse.getMetrics()) {
MetricDataset metricDataset = new MetricDataset(metric, dataset);
MetricConfigDTO metricConfig = CACHE_REGISTRY_INSTANCE.getMetricConfigCache().get(metricDataset);
Long metricId = metricConfig.getId();
GenericResponse response = tabularViewResponse.getData().get(metric);
MetricSummary metricSummary = new MetricSummary();
metricSummary.setMetricId(metricId);
metricSummary.setMetricName(metricConfig.getName());
metricSummary.setMetricAlias(metricConfig.getAlias());
String[] responseData = response.getResponseData().get(0);
double baselineValue = Double.valueOf(responseData[0]);
double curentvalue = Double.valueOf(responseData[1]);
double percentageChange = (curentvalue - baselineValue) * 100 / baselineValue;
metricSummary.setBaselineValue(baselineValue);
metricSummary.setCurrentValue(curentvalue);
metricSummary.setWowPercentageChange(percentageChange);
AnomaliesSummary anomaliesSummary = anomaliesResoure.getAnomalyCountForMetricInRange(metricId, currentStart, currentEnd);
metricSummary.setAnomaliesSummary(anomaliesSummary);
metricsSummary.add(metricSummary);
}
} catch (Exception e) {
LOG.error("Exception while processing /data/tabular call", e);
}
}
return metricsSummary;
}
Aggregations