Examples with TimeAggregation org.hypertrace.gateway.service.v1.common.TimeAggregation used on opensource projects

Example 1 with TimeAggregation

use of org.hypertrace.gateway.service.v1.common.TimeAggregation in project gateway-service by hypertrace.

the class QueryServiceEntityFetcher method buildTimeSeriesQueryRequest.

private QueryRequest buildTimeSeriesQueryRequest(EntitiesRequest entitiesRequest, EntitiesRequestContext context, long periodSecs, List<TimeAggregation> timeAggregationBatch, List<String> idColumns, String timeColumn) {
    long alignedStartTime = QueryExpressionUtil.alignToPeriodBoundary(entitiesRequest.getStartTimeMillis(), periodSecs, true);
    long alignedEndTime = QueryExpressionUtil.alignToPeriodBoundary(entitiesRequest.getEndTimeMillis(), periodSecs, false);
    EntitiesRequest timeAlignedEntitiesRequest = EntitiesRequest.newBuilder(entitiesRequest).setStartTimeMillis(alignedStartTime).setEndTimeMillis(alignedEndTime).build();
    QueryRequest.Builder builder = QueryRequest.newBuilder();
    timeAggregationBatch.forEach(e -> builder.addSelection(QueryAndGatewayDtoConverter.convertToQueryExpression(e.getAggregation())));
    Filter.Builder queryFilter = constructQueryServiceFilter(timeAlignedEntitiesRequest, context, idColumns);
    builder.setFilter(queryFilter);
    // First group by the id columns.
    builder.addAllGroupBy(idColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList()));
    // Secondary grouping is on time.
    builder.addGroupBy(createTimeColumnGroupByExpression(timeColumn, periodSecs));
    // Pinot truncates the GroupBy results to 10 when there is no limit explicitly but
    // here we neither want the results to be truncated nor apply the limit coming from client.
    // We would like to get all entities based on filters so we set the limit to a high value.
    // TODO: Figure out a reasonable computed limit instead of this hardcoded one. Probably
    // requested limit * expected max number of time series buckets
    builder.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT);
    return builder.build();
}

Example 2 with TimeAggregation

use of org.hypertrace.gateway.service.v1.common.TimeAggregation in project gateway-service by hypertrace.

the class QueryServiceEntityFetcher method getTimeAggregatedMetrics.

@Override
public EntityFetcherResponse getTimeAggregatedMetrics(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    // No need to make execute the rest of this if there are no TimeAggregations in the request.
    if (entitiesRequest.getTimeAggregationCount() == 0) {
        return new EntityFetcherResponse();
    }
    // Only supported filter is entityIds IN ["id1", "id2", "id3"]
    List<String> idColumns = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    String timeColumn = AttributeMetadataUtil.getTimestampAttributeId(attributeMetadataProvider, requestContext, entitiesRequest.getEntityType());
    Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
    Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
    entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
    entitiesRequest.getTimeAggregationList().forEach(timeAggregation -> requestContext.mapAliasToTimeAggregation(timeAggregation.getAggregation().getFunction().getAlias(), timeAggregation));
    // First group the Aggregations based on the period so that we can issue separate queries
    // to QueryService for each different Period.
    Collection<List<TimeAggregation>> result = entitiesRequest.getTimeAggregationList().stream().collect(Collectors.groupingBy(TimeAggregation::getPeriod)).values();
    Map<EntityKey, Map<String, MetricSeries.Builder>> entityMetricSeriesMap = new LinkedHashMap<>();
    for (List<TimeAggregation> batch : result) {
        Period period = batch.get(0).getPeriod();
        ChronoUnit unit = ChronoUnit.valueOf(period.getUnit());
        long periodSecs = Duration.of(period.getValue(), unit).getSeconds();
        QueryRequest request = buildTimeSeriesQueryRequest(entitiesRequest, requestContext, periodSecs, batch, idColumns, timeColumn);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Sending time series queryRequest to query service: ======== \n {}", request.toString());
        }
        Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(request, requestContext.getHeaders(), requestTimeout);
        while (resultSetChunkIterator.hasNext()) {
            ResultSetChunk chunk = resultSetChunkIterator.next();
            if (LOG.isDebugEnabled()) {
                LOG.debug("Received chunk: " + chunk.toString());
            }
            if (chunk.getRowCount() < 1) {
                break;
            }
            if (!chunk.hasResultSetMetadata()) {
                LOG.warn("Chunk doesn't have result metadata so couldn't process the response.");
                break;
            }
            for (Row row : chunk.getRowList()) {
                // Construct the entity id from the entityIdAttributeIds columns
                EntityKey entityKey = EntityKey.of(IntStream.range(0, idColumns.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
                Map<String, MetricSeries.Builder> metricSeriesMap = entityMetricSeriesMap.computeIfAbsent(entityKey, k -> new LinkedHashMap<>());
                Interval.Builder intervalBuilder = Interval.newBuilder();
                // Second column is the time column
                Value value = QueryAndGatewayDtoConverter.convertQueryValueToGatewayValue(row.getColumn(idColumns.size()));
                if (value.getValueType() == ValueType.STRING) {
                    long startTime = Long.parseLong(value.getString());
                    long endTime = startTime + TimeUnit.SECONDS.toMillis(periodSecs);
                    intervalBuilder.setStartTimeMillis(startTime);
                    intervalBuilder.setEndTimeMillis(endTime);
                    for (int i = idColumns.size() + 1; i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                        ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
                        TimeAggregation timeAggregation = requestContext.getTimeAggregationByAlias(metadata.getColumnName());
                        if (timeAggregation == null) {
                            LOG.warn("Couldn't find an aggregate for column: {}", metadata.getColumnName());
                            continue;
                        }
                        FunctionType functionType = timeAggregation.getAggregation().getFunction().getFunction();
                        AttributeMetadata functionAttributeMetadata = resultKeyToAttributeMetadataMap.get(metadata.getColumnName());
                        Value convertedValue = QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue(MetricAggregationFunctionUtil.getValueTypeForFunctionType(functionType, functionAttributeMetadata), resultKeyToAttributeMetadataMap, metadata, row.getColumn(i));
                        List<org.hypertrace.gateway.service.v1.common.Expression> healthExpressions = timeAggregation.getAggregation().getFunction().getArgumentsList().stream().filter(org.hypertrace.gateway.service.v1.common.Expression::hasHealth).collect(Collectors.toList());
                        Preconditions.checkArgument(healthExpressions.size() <= 1);
                        Health health = Health.NOT_COMPUTED;
                        MetricSeries.Builder seriesBuilder = metricSeriesMap.computeIfAbsent(metadata.getColumnName(), k -> getMetricSeriesBuilder(timeAggregation));
                        seriesBuilder.addValue(Interval.newBuilder(intervalBuilder.build()).setValue(convertedValue).setHealth(health));
                    }
                } else {
                    LOG.warn("Was expecting STRING values only but received valueType: {}", value.getValueType());
                }
            }
        }
    }
    Map<EntityKey, Entity.Builder> resultMap = new LinkedHashMap<>();
    for (Map.Entry<EntityKey, Map<String, MetricSeries.Builder>> entry : entityMetricSeriesMap.entrySet()) {
        Entity.Builder entityBuilder = Entity.newBuilder().setEntityType(entitiesRequest.getEntityType()).setId(entry.getKey().toString()).putAllMetricSeries(entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> getSortedMetricSeries(e.getValue()))));
        for (int i = 0; i < idColumns.size(); i++) {
            entityBuilder.putAttribute(idColumns.get(i), Value.newBuilder().setString(entry.getKey().getAttributes().get(i)).setValueType(ValueType.STRING).build());
        }
        resultMap.put(entry.getKey(), entityBuilder);
    }
    return new EntityFetcherResponse(resultMap);
}

Example 3 with TimeAggregation

use of org.hypertrace.gateway.service.v1.common.TimeAggregation in project gateway-service by hypertrace.

the class BaselineServiceImpl method getBaselineForEntities.

public BaselineEntitiesResponse getBaselineForEntities(String tenantId, BaselineEntitiesRequest originalRequest, Map<String, String> requestHeaders) {
    BaselineRequestContext requestContext = getRequestContext(tenantId, requestHeaders, originalRequest);
    Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, originalRequest.getEntityType());
    baselineEntitiesRequestValidator.validate(originalRequest, attributeMetadataMap);
    String timeColumn = AttributeMetadataUtil.getTimestampAttributeId(attributeMetadataProvider, requestContext, originalRequest.getEntityType());
    Map<String, BaselineEntity> baselineEntityAggregatedMetricsMap = new HashMap<>();
    Map<String, BaselineEntity> baselineEntityTimeSeriesMap = new HashMap<>();
    if (originalRequest.getBaselineAggregateRequestCount() > 0) {
        // Aggregated Functions data
        Period aggTimePeriod = getPeriod(originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis());
        long periodSecs = getPeriodInSecs(aggTimePeriod);
        long aggStartTime = originalRequest.getStartTimeMillis();
        long aggEndTime = originalRequest.getEndTimeMillis();
        List<TimeAggregation> timeAggregations = getTimeAggregationsForAggregateExpr(originalRequest, aggStartTime, aggEndTime);
        updateAliasMap(requestContext, timeAggregations);
        // Take more data to calculate baseline and standard deviation.
        long seriesStartTime = getUpdatedStartTime(aggStartTime, aggEndTime);
        long seriesEndTime = aggStartTime;
        List<String> entityIdAttributes = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, originalRequest.getEntityType());
        QueryRequest aggQueryRequest = baselineServiceQueryParser.getQueryRequest(seriesStartTime, seriesEndTime, originalRequest.getEntityIdsList(), timeColumn, timeAggregations, periodSecs, entityIdAttributes);
        Iterator<ResultSetChunk> aggResponseChunkIterator = baselineServiceQueryExecutor.executeQuery(requestHeaders, aggQueryRequest);
        BaselineEntitiesResponse aggEntitiesResponse = baselineServiceQueryParser.parseQueryResponse(aggResponseChunkIterator, requestContext, entityIdAttributes.size(), originalRequest.getEntityType(), aggStartTime, aggEndTime);
        baselineEntityAggregatedMetricsMap = getEntitiesMapFromAggResponse(aggEntitiesResponse);
    }
    // Time Series data
    if (originalRequest.getBaselineMetricSeriesRequestCount() > 0) {
        Period timeSeriesPeriod = getTimeSeriesPeriod(originalRequest.getBaselineMetricSeriesRequestList());
        long periodSecs = getPeriodInSecs(timeSeriesPeriod);
        long alignedStartTime = QueryExpressionUtil.alignToPeriodBoundary(originalRequest.getStartTimeMillis(), periodSecs, true);
        long alignedEndTime = QueryExpressionUtil.alignToPeriodBoundary(originalRequest.getEndTimeMillis(), periodSecs, false);
        List<TimeAggregation> timeAggregations = getTimeAggregationsForTimeSeriesExpr(originalRequest);
        long seriesStartTime = getUpdatedStartTime(alignedStartTime, alignedEndTime);
        long seriesEndTime = alignedStartTime;
        List<String> entityIdAttributes = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, originalRequest.getEntityType());
        QueryRequest timeSeriesQueryRequest = baselineServiceQueryParser.getQueryRequest(seriesStartTime, seriesEndTime, originalRequest.getEntityIdsList(), timeColumn, timeAggregations, periodSecs, entityIdAttributes);
        Iterator<ResultSetChunk> timeSeriesChunkIterator = baselineServiceQueryExecutor.executeQuery(requestHeaders, timeSeriesQueryRequest);
        BaselineEntitiesResponse timeSeriesEntitiesResponse = baselineServiceQueryParser.parseQueryResponse(timeSeriesChunkIterator, requestContext, entityIdAttributes.size(), originalRequest.getEntityType(), alignedStartTime, alignedEndTime);
        baselineEntityTimeSeriesMap = getEntitiesMapFromTimeSeriesResponse(timeSeriesEntitiesResponse, alignedStartTime, alignedEndTime, periodSecs);
    }
    return mergeEntities(baselineEntityAggregatedMetricsMap, baselineEntityTimeSeriesMap);
}

Example 4 with TimeAggregation

use of org.hypertrace.gateway.service.v1.common.TimeAggregation in project gateway-service by hypertrace.

the class BaselineServiceImpl method getTimeAggregationsForTimeSeriesExpr.

private List<TimeAggregation> getTimeAggregationsForTimeSeriesExpr(BaselineEntitiesRequest originalRequest) {
    List<BaselineTimeAggregation> timeSeriesList = originalRequest.getBaselineMetricSeriesRequestList();
    List<TimeAggregation> timeAggregations = new ArrayList<>();
    for (BaselineTimeAggregation timeAggregation : timeSeriesList) {
        Expression aggregation = Expression.newBuilder().setFunction(timeAggregation.getAggregation()).build();
        TimeAggregation agg = TimeAggregation.newBuilder().setAggregation(aggregation).setPeriod(timeAggregation.getPeriod()).build();
        timeAggregations.add(agg);
    }
    return timeAggregations;
}

Example 5 with TimeAggregation

use of org.hypertrace.gateway.service.v1.common.TimeAggregation in project gateway-service by hypertrace.

the class BaselineServiceImpl method getTimeAggregationsForAggregateExpr.

private List<TimeAggregation> getTimeAggregationsForAggregateExpr(BaselineEntitiesRequest originalRequest, long alignedStartTime, long alignedEndTime) {
    List<FunctionExpression> aggregateList = originalRequest.getBaselineAggregateRequestList();
    List<TimeAggregation> timeAggregationList = new ArrayList<>();
    for (FunctionExpression function : aggregateList) {
        TimeAggregation timeAggregation = getAggregationFunction(function, alignedStartTime, alignedEndTime);
        timeAggregationList.add(timeAggregation);
    }
    return timeAggregationList;
}