Examples with MetricConfigDTO com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO used on opensource projects

Example 16 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class HeatMapViewHandler method process.

@Override
public HeatMapViewResponse process(HeatMapViewRequest request) throws Exception {
    // query 1 for everything from baseline start to baseline end
    // query 2 for everything from current start to current end
    // for each dimension group by top 100
    // query 1 for everything from baseline start to baseline end
    // query for everything from current start to current end
    List<String> expressionNames = new ArrayList<>();
    Map<String, String> metricExpressions = new HashMap<>();
    Set<String> metricOrExpressionNames = new HashSet<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        expressionNames.add(expression.getExpressionName());
        metricExpressions.put(expression.getExpressionName(), expression.getExpression());
        metricOrExpressionNames.add(expression.getExpressionName());
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        for (MetricFunction function : metricFunctions) {
            metricOrExpressionNames.add(function.getMetricName());
        }
    }
    Map<String, HeatMap.Builder> data = new HashMap<>();
    TimeOnTimeComparisonRequest comparisonRequest = generateTimeOnTimeComparisonRequest(request);
    List<String> groupByDimensions = comparisonRequest.getGroupByDimensions();
    final TimeOnTimeComparisonHandler handler = new TimeOnTimeComparisonHandler(queryCache);
    // we are tracking per dimension, to validate that its the same for each dimension
    Map<String, Map<String, Double>> baselineTotalPerMetricAndDimension = new HashMap<>();
    Map<String, Map<String, Double>> currentTotalPerMetricAndDimension = new HashMap<>();
    for (String metricOrExpressionName : metricOrExpressionNames) {
        Map<String, Double> baselineTotalMap = new HashMap<>();
        Map<String, Double> currentTotalMap = new HashMap<>();
        baselineTotalPerMetricAndDimension.put(metricOrExpressionName, baselineTotalMap);
        currentTotalPerMetricAndDimension.put(metricOrExpressionName, currentTotalMap);
        for (String dimension : groupByDimensions) {
            baselineTotalMap.put(dimension, 0d);
            currentTotalMap.put(dimension, 0d);
        }
    }
    List<Future<TimeOnTimeComparisonResponse>> timeOnTimeComparisonResponsesFutures = getTimeOnTimeComparisonResponses(groupByDimensions, comparisonRequest, handler);
    for (int groupByDimensionId = 0; groupByDimensionId < groupByDimensions.size(); groupByDimensionId++) {
        String groupByDimension = groupByDimensions.get(groupByDimensionId);
        TimeOnTimeComparisonResponse response = timeOnTimeComparisonResponsesFutures.get(groupByDimensionId).get();
        int numRows = response.getNumRows();
        for (int i = 0; i < numRows; i++) {
            Row row = response.getRow(i);
            String dimensionValue = row.getDimensionValue();
            Map<String, Metric> metricMap = new HashMap<>();
            for (Metric metric : row.getMetrics()) {
                metricMap.put(metric.getMetricName(), metric);
            }
            for (Metric metric : row.getMetrics()) {
                String metricName = metric.getMetricName();
                // update the baselineTotal and current total
                Map<String, Double> baselineTotalMap = baselineTotalPerMetricAndDimension.get(metricName);
                Map<String, Double> currentTotalMap = currentTotalPerMetricAndDimension.get(metricName);
                baselineTotalMap.put(groupByDimension, baselineTotalMap.get(groupByDimension) + metric.getBaselineValue());
                currentTotalMap.put(groupByDimension, currentTotalMap.get(groupByDimension) + metric.getCurrentValue());
                if (!expressionNames.contains(metricName)) {
                    continue;
                }
                String dataKey = metricName + "." + groupByDimension;
                HeatMap.Builder heatMapBuilder = data.get(dataKey);
                if (heatMapBuilder == null) {
                    heatMapBuilder = new HeatMap.Builder(groupByDimension);
                    data.put(dataKey, heatMapBuilder);
                }
                MetricDataset metricDataset = new MetricDataset(metricName, comparisonRequest.getCollectionName());
                MetricConfigDTO metricConfig = CACHE_REGISTRY.getMetricConfigCache().get(metricDataset);
                if (StringUtils.isNotBlank(metricConfig.getCellSizeExpression())) {
                    String metricExpression = metricExpressions.get(metricName);
                    String[] tokens = metricExpression.split(RATIO_SEPARATOR);
                    String numerator = tokens[0];
                    String denominator = tokens[1];
                    Metric numeratorMetric = metricMap.get(numerator);
                    Metric denominatorMetric = metricMap.get(denominator);
                    Double numeratorBaseline = numeratorMetric == null ? 0 : numeratorMetric.getBaselineValue();
                    Double numeratorCurrent = numeratorMetric == null ? 0 : numeratorMetric.getCurrentValue();
                    Double denominatorBaseline = denominatorMetric == null ? 0 : denominatorMetric.getBaselineValue();
                    Double denominatorCurrent = denominatorMetric == null ? 0 : denominatorMetric.getCurrentValue();
                    Map<String, Double> context = new HashMap<>();
                    context.put(numerator, numeratorCurrent);
                    context.put(denominator, denominatorCurrent);
                    String cellSizeExpression = metricConfig.getCellSizeExpression();
                    Double cellSize = MetricExpression.evaluateExpression(cellSizeExpression, context);
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue(), cellSize, cellSizeExpression, numeratorBaseline, denominatorBaseline, numeratorCurrent, denominatorCurrent);
                } else {
                    heatMapBuilder.addCell(dimensionValue, metric.getBaselineValue(), metric.getCurrentValue());
                }
            }
        }
    }
    ResponseSchema schema = new ResponseSchema();
    String[] columns = HeatMapCell.columns();
    for (int i = 0; i < columns.length; i++) {
        String column = columns[i];
        schema.add(column, i);
    }
    Info summary = new Info();
    Map<String, GenericResponse> heatMapViewResponseData = new HashMap<>();
    for (MetricExpression expression : request.getMetricExpressions()) {
        List<MetricFunction> metricFunctions = expression.computeMetricFunctions();
        Double baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        Double currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        // check if its derived
        if (metricFunctions.size() > 1) {
            Map<String, Double> baselineContext = new HashMap<>();
            Map<String, Double> currentContext = new HashMap<>();
            for (String metricOrExpression : metricOrExpressionNames) {
                baselineContext.put(metricOrExpression, baselineTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
                currentContext.put(metricOrExpression, currentTotalPerMetricAndDimension.get(metricOrExpression).values().iterator().next());
            }
            baselineTotal = MetricExpression.evaluateExpression(expression, baselineContext);
            currentTotal = MetricExpression.evaluateExpression(expression, currentContext);
        } else {
            baselineTotal = baselineTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
            currentTotal = currentTotalPerMetricAndDimension.get(expression.getExpressionName()).values().iterator().next();
        }
        summary.addSimpleField("baselineStart", Long.toString(comparisonRequest.getBaselineStart().getMillis()));
        summary.addSimpleField("baselineEnd", Long.toString(comparisonRequest.getBaselineEnd().getMillis()));
        summary.addSimpleField("currentStart", Long.toString(comparisonRequest.getCurrentStart().getMillis()));
        summary.addSimpleField("currentEnd", Long.toString(comparisonRequest.getCurrentEnd().getMillis()));
        summary.addSimpleField("baselineTotal", HeatMapCell.format(baselineTotal));
        summary.addSimpleField("currentTotal", HeatMapCell.format(currentTotal));
        summary.addSimpleField("deltaChange", HeatMapCell.format(currentTotal - baselineTotal));
        summary.addSimpleField("deltaPercentage", HeatMapCell.format((currentTotal - baselineTotal) * 100.0 / baselineTotal));
    }
    for (Entry<String, HeatMap.Builder> entry : data.entrySet()) {
        String dataKey = entry.getKey();
        GenericResponse heatMapResponse = new GenericResponse();
        List<String[]> heatMapResponseData = new ArrayList<>();
        HeatMap.Builder builder = entry.getValue();
        HeatMap heatMap = builder.build();
        for (HeatMapCell cell : heatMap.heatMapCells) {
            String[] newRowData = cell.toArray();
            heatMapResponseData.add(newRowData);
        }
        heatMapResponse.setSchema(schema);
        heatMapResponse.setResponseData(heatMapResponseData);
        heatMapViewResponseData.put(dataKey, heatMapResponse);
    }
    HeatMapViewResponse heatMapViewResponse = new HeatMapViewResponse();
    heatMapViewResponse.setMetrics(expressionNames);
    heatMapViewResponse.setDimensions(groupByDimensions);
    heatMapViewResponse.setData(heatMapViewResponseData);
    heatMapViewResponse.setMetricExpression(metricExpressions);
    heatMapViewResponse.setSummary(summary);
    return heatMapViewResponse;
}

Example 17 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class TimeSeriesResource method getContributorDataForDimension.

private TimeSeriesCompareMetricView getContributorDataForDimension(long metricId, long currentStart, long currentEnd, long baselineStart, long baselineEnd, String dimension, String filters, String granularity) {
    MetricConfigDTO metricConfigDTO = metricConfigDAO.findById(metricId);
    TimeSeriesCompareMetricView timeSeriesCompareMetricView = new TimeSeriesCompareMetricView(metricConfigDTO.getName(), metricId, currentStart, currentEnd);
    try {
        String dataset = metricConfigDTO.getDataset();
        ContributorViewRequest request = new ContributorViewRequest();
        request.setCollection(dataset);
        MetricExpression metricExpression = ThirdEyeUtils.getMetricExpressionFromMetricConfig(metricConfigDTO);
        request.setMetricExpressions(Arrays.asList(metricExpression));
        DateTimeZone timeZoneForCollection = Utils.getDataTimeZone(dataset);
        request.setBaselineStart(new DateTime(baselineStart, timeZoneForCollection));
        request.setBaselineEnd(new DateTime(baselineEnd, timeZoneForCollection));
        request.setCurrentStart(new DateTime(currentStart, timeZoneForCollection));
        request.setCurrentEnd(new DateTime(currentEnd, timeZoneForCollection));
        request.setTimeGranularity(Utils.getAggregationTimeGranularity(granularity, dataset));
        if (filters != null && !filters.isEmpty()) {
            filters = URLDecoder.decode(filters, "UTF-8");
            request.setFilters(ThirdEyeUtils.convertToMultiMap(filters));
        }
        request.setGroupByDimensions(Arrays.asList(dimension));
        ContributorViewHandler handler = new ContributorViewHandler(queryCache);
        ContributorViewResponse response = handler.process(request);
        // Assign the time buckets
        List<Long> timeBucketsCurrent = new ArrayList<>();
        List<Long> timeBucketsBaseline = new ArrayList<>();
        timeSeriesCompareMetricView.setTimeBucketsCurrent(timeBucketsCurrent);
        timeSeriesCompareMetricView.setTimeBucketsBaseline(timeBucketsBaseline);
        Map<String, ValuesContainer> subDimensionValuesMap = new LinkedHashMap<>();
        timeSeriesCompareMetricView.setSubDimensionContributionMap(subDimensionValuesMap);
        int timeBuckets = response.getTimeBuckets().size();
        // this is for over all values
        ValuesContainer vw = new ValuesContainer();
        subDimensionValuesMap.put(ALL, vw);
        vw.setCurrentValues(new double[timeBuckets]);
        vw.setBaselineValues(new double[timeBuckets]);
        vw.setPercentageChange(new String[timeBuckets]);
        vw.setCumulativeCurrentValues(new double[timeBuckets]);
        vw.setCumulativeBaselineValues(new double[timeBuckets]);
        vw.setCumulativePercentageChange(new String[timeBuckets]);
        // lets find the indices
        int subDimensionIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("dimensionValue");
        int currentValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("currentValue");
        int baselineValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("baselineValue");
        int percentageChangeIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("percentageChange");
        int cumCurrentValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativeCurrentValue");
        int cumBaselineValueIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativeBaselineValue");
        int cumPercentageChangeIndex = response.getResponseData().getSchema().getColumnsToIndexMapping().get("cumulativePercentageChange");
        // populate current and baseline time buckets
        for (int i = 0; i < timeBuckets; i++) {
            TimeBucket tb = response.getTimeBuckets().get(i);
            timeBucketsCurrent.add(tb.getCurrentStart());
            timeBucketsBaseline.add(tb.getBaselineStart());
        }
        // set current and baseline values for sub dimensions
        for (int i = 0; i < response.getResponseData().getResponseData().size(); i++) {
            String[] data = response.getResponseData().getResponseData().get(i);
            String subDimension = data[subDimensionIndex];
            Double currentVal = Double.valueOf(data[currentValueIndex]);
            Double baselineVal = Double.valueOf(data[baselineValueIndex]);
            Double percentageChangeVal = Double.valueOf(data[percentageChangeIndex]);
            Double cumCurrentVal = Double.valueOf(data[cumCurrentValueIndex]);
            Double cumBaselineVal = Double.valueOf(data[cumBaselineValueIndex]);
            Double cumPercentageChangeVal = Double.valueOf(data[cumPercentageChangeIndex]);
            int index = i % timeBuckets;
            // set overAll values
            vw.getCurrentValues()[index] += currentVal;
            vw.getBaselineValues()[index] += baselineVal;
            vw.getCumulativeCurrentValues()[index] += cumCurrentVal;
            vw.getCumulativeBaselineValues()[index] += cumBaselineVal;
            // set individual sub-dimension values
            if (!subDimensionValuesMap.containsKey(subDimension)) {
                ValuesContainer subDimVals = new ValuesContainer();
                subDimVals.setCurrentValues(new double[timeBuckets]);
                subDimVals.setBaselineValues(new double[timeBuckets]);
                subDimVals.setPercentageChange(new String[timeBuckets]);
                subDimVals.setCumulativeCurrentValues(new double[timeBuckets]);
                subDimVals.setCumulativeBaselineValues(new double[timeBuckets]);
                subDimVals.setCumulativePercentageChange(new String[timeBuckets]);
                subDimensionValuesMap.put(subDimension, subDimVals);
            }
            subDimensionValuesMap.get(subDimension).getCurrentValues()[index] = currentVal;
            subDimensionValuesMap.get(subDimension).getBaselineValues()[index] = baselineVal;
            subDimensionValuesMap.get(subDimension).getPercentageChange()[index] = String.format(DECIMAL_FORMAT, percentageChangeVal);
            subDimensionValuesMap.get(subDimension).getCumulativeCurrentValues()[index] = cumCurrentVal;
            subDimensionValuesMap.get(subDimension).getCumulativeBaselineValues()[index] = cumBaselineVal;
            subDimensionValuesMap.get(subDimension).getCumulativePercentageChange()[index] = String.format(DECIMAL_FORMAT, cumPercentageChangeVal);
        }
        // TODO : compute cumulative values for all
        for (int i = 0; i < vw.getCurrentValues().length; i++) {
            vw.getPercentageChange()[i] = String.format(DECIMAL_FORMAT, getPercentageChange(vw.getCurrentValues()[i], vw.getBaselineValues()[i]));
            vw.getCumulativePercentageChange()[i] = String.format(DECIMAL_FORMAT, getPercentageChange(vw.getCumulativeCurrentValues()[i], vw.getCumulativeBaselineValues()[i]));
        }
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new WebApplicationException(e);
    }
    return timeSeriesCompareMetricView;
}

Example 18 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class MetricConfigManagerImpl method findByMetricAndDataset.

@Override
public MetricConfigDTO findByMetricAndDataset(String metricName, String dataset) {
    Predicate datasetPredicate = Predicate.EQ("dataset", dataset);
    Predicate metricNamePredicate = Predicate.EQ("name", metricName);
    List<MetricConfigBean> list = genericPojoDao.get(Predicate.AND(datasetPredicate, metricNamePredicate), MetricConfigBean.class);
    MetricConfigDTO result = null;
    if (CollectionUtils.isNotEmpty(list)) {
        result = MODEL_MAPPER.map(list.get(0), MetricConfigDTO.class);
    }
    return result;
}

Example 19 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class MetricConfigManagerImpl method findByAliasAndDataset.

@Override
public MetricConfigDTO findByAliasAndDataset(String alias, String dataset) {
    Predicate datasetPredicate = Predicate.EQ("dataset", dataset);
    Predicate aliasPredicate = Predicate.EQ("alias", alias);
    List<MetricConfigBean> list = genericPojoDao.get(Predicate.AND(datasetPredicate, aliasPredicate), MetricConfigBean.class);
    MetricConfigDTO result = null;
    if (CollectionUtils.isNotEmpty(list)) {
        result = MODEL_MAPPER.map(list.get(0), MetricConfigDTO.class);
    }
    return result;
}

Example 20 with MetricConfigDTO

use of com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO in project pinot by linkedin.

the class MetricConfigManagerImpl method findByMetricName.

public List<MetricConfigDTO> findByMetricName(String metricName) {
    Predicate metricNamePredicate = Predicate.EQ("name", metricName);
    List<MetricConfigBean> list = genericPojoDao.get(metricNamePredicate, MetricConfigBean.class);
    List<MetricConfigDTO> result = new ArrayList<>();
    for (MetricConfigBean abstractBean : list) {
        MetricConfigDTO dto = MODEL_MAPPER.map(abstractBean, MetricConfigDTO.class);
        result.add(dto);
    }
    return result;
}