use of org.hypertrace.core.query.service.api.QueryRequest in project gateway-service by hypertrace.
the class EntityInteractionsFetcher method buildQueryRequests.
@VisibleForTesting
Map<String, QueryRequest> buildQueryRequests(long startTime, long endTime, String spaceId, String entityType, InteractionsRequest interactionsRequest, Set<EntityKey> entityIds, boolean incoming, RequestContext requestContext) {
Set<String> entityTypes = getOtherEntityTypes(interactionsRequest.getFilter());
if (entityTypes.isEmpty()) {
return Collections.emptyMap();
}
QueryRequest.Builder builder = QueryRequest.newBuilder();
// Filter should include the timestamp filters from parent request first
Filter.Builder filterBuilder = Filter.newBuilder().setOperator(Operator.AND).addChildFilter(QueryRequestUtil.createBetweenTimesFilter(AttributeMetadataUtil.getTimestampAttributeId(metadataProvider, requestContext, SCOPE), startTime, endTime));
this.buildSpaceQueryFilterIfNeeded(requestContext, spaceId).ifPresent(filterBuilder::addChildFilter);
List<String> idColumns = getEntityIdColumnsFromInteraction(DomainEntityType.valueOf(entityType), !incoming);
// Add a filter on the entityIds
filterBuilder.addChildFilter(createFilterForEntityKeys(idColumns, entityIds));
// Group by the entity id column first, then the other end entity type for the interaction.
List<org.hypertrace.core.query.service.api.Expression> idExpressions = idColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList());
builder.addAllGroupBy(idExpressions);
List<org.hypertrace.core.query.service.api.Expression> selections = new ArrayList<>();
for (Expression expression : interactionsRequest.getSelectionList()) {
// Ignore the predefined selections because they're handled specially.
if (ExpressionReader.isSimpleAttributeSelection(expression) && SELECTIONS_TO_IGNORE.contains(ExpressionReader.getAttributeIdFromAttributeSelection(expression).orElseThrow())) {
continue;
}
// Selection should have metrics and attributes that were requested
selections.add(QueryAndGatewayDtoConverter.convertToQueryExpression(expression).build());
}
// so we add count(*) as a dummy placeholder if there are no explicit selectors.
if (selections.isEmpty()) {
selections.add(QueryRequestUtil.createCountByColumnSelection(Optional.ofNullable(idColumns.get(0)).orElseThrow()));
}
QueryRequest protoType = builder.build();
Filter protoTypeFilter = filterBuilder.build();
Map<String, QueryRequest> queryRequests = new HashMap<>();
// response time.
for (String e : entityTypes) {
DomainEntityType otherEntityType = DomainEntityType.valueOf(e.toUpperCase());
// Get the filters from the interactions request to 'AND' them with the timestamp filter.
Filter.Builder filterCopy = Filter.newBuilder(protoTypeFilter);
filterCopy.addChildFilter(convertToQueryFilter(interactionsRequest.getFilter(), otherEntityType));
QueryRequest.Builder builderCopy = QueryRequest.newBuilder(protoType);
builderCopy.setFilter(filterCopy);
List<String> otherEntityIdColumns = getEntityIdColumnsFromInteraction(otherEntityType, incoming);
List<org.hypertrace.core.query.service.api.Expression> otherIdExpressions = otherEntityIdColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList());
builderCopy.addAllGroupBy(otherIdExpressions);
// Add all selections in the correct order. First id, then other entity id and finally
// the remaining selections.
builderCopy.addAllSelection(idExpressions);
builderCopy.addAllSelection(otherIdExpressions);
selections.forEach(builderCopy::addSelection);
int limit = interactionsRequest.getLimit();
if (limit > 0) {
builderCopy.setLimit(limit);
} else {
builderCopy.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT);
}
queryRequests.put(e, builderCopy.build());
}
return queryRequests;
}
use of org.hypertrace.core.query.service.api.QueryRequest in project gateway-service by hypertrace.
the class QueryServiceEntityFetcher method buildTimeSeriesQueryRequest.
private QueryRequest buildTimeSeriesQueryRequest(EntitiesRequest entitiesRequest, EntitiesRequestContext context, long periodSecs, List<TimeAggregation> timeAggregationBatch, List<String> idColumns, String timeColumn) {
long alignedStartTime = QueryExpressionUtil.alignToPeriodBoundary(entitiesRequest.getStartTimeMillis(), periodSecs, true);
long alignedEndTime = QueryExpressionUtil.alignToPeriodBoundary(entitiesRequest.getEndTimeMillis(), periodSecs, false);
EntitiesRequest timeAlignedEntitiesRequest = EntitiesRequest.newBuilder(entitiesRequest).setStartTimeMillis(alignedStartTime).setEndTimeMillis(alignedEndTime).build();
QueryRequest.Builder builder = QueryRequest.newBuilder();
timeAggregationBatch.forEach(e -> builder.addSelection(QueryAndGatewayDtoConverter.convertToQueryExpression(e.getAggregation())));
Filter.Builder queryFilter = constructQueryServiceFilter(timeAlignedEntitiesRequest, context, idColumns);
builder.setFilter(queryFilter);
// First group by the id columns.
builder.addAllGroupBy(idColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList()));
// Secondary grouping is on time.
builder.addGroupBy(createTimeColumnGroupByExpression(timeColumn, periodSecs));
// Pinot truncates the GroupBy results to 10 when there is no limit explicitly but
// here we neither want the results to be truncated nor apply the limit coming from client.
// We would like to get all entities based on filters so we set the limit to a high value.
// TODO: Figure out a reasonable computed limit instead of this hardcoded one. Probably
// requested limit * expected max number of time series buckets
builder.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT);
return builder.build();
}
use of org.hypertrace.core.query.service.api.QueryRequest in project gateway-service by hypertrace.
the class QueryServiceEntityFetcher method getEntities.
@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
// Validate EntitiesRequest
entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
List<org.hypertrace.gateway.service.v1.common.Expression> aggregates = ExpressionReader.getFunctionExpressions(entitiesRequest.getSelectionList());
Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
QueryRequest.Builder builder = constructSelectionQuery(requestContext, entitiesRequest, entityIdAttributeIds, aggregates);
adjustLimitAndOffset(builder, entitiesRequest.getLimit(), entitiesRequest.getOffset());
if (!entitiesRequest.getOrderByList().isEmpty()) {
// Order by from the request.
builder.addAllOrderBy(QueryAndGatewayDtoConverter.convertToQueryOrderByExpressions(entitiesRequest.getOrderByList()));
}
QueryRequest queryRequest = builder.build();
LOG.debug("Sending Query to Query Service ======== \n {}", queryRequest);
Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(queryRequest, requestContext.getHeaders(), requestTimeout);
// We want to retain the order as returned from the respective source. Hence using a
// LinkedHashMap
Map<EntityKey, Entity.Builder> entityBuilders = new LinkedHashMap<>();
while (resultSetChunkIterator.hasNext()) {
ResultSetChunk chunk = resultSetChunkIterator.next();
LOG.debug("Received chunk: {}", chunk);
if (chunk.getRowCount() < 1) {
break;
}
for (Row row : chunk.getRowList()) {
// Construct the entity id from the entityIdAttributeIds columns
EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
entityBuilder.setEntityType(entitiesRequest.getEntityType());
entityBuilder.setId(entityKey.toString());
// as post processing.
for (int i = 0; i < entityIdAttributeIds.size(); i++) {
entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
}
requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
org.hypertrace.core.query.service.api.Value columnValue = row.getColumn(i);
buildEntity(entityBuilder, requestContext, metadata, columnValue, resultKeyToAttributeMetadataMap, aggregates.isEmpty());
}
}
}
return new EntityFetcherResponse(entityBuilders);
}
use of org.hypertrace.core.query.service.api.QueryRequest in project gateway-service by hypertrace.
the class QueryServiceEntityFetcher method getTimeAggregatedMetrics.
@Override
public EntityFetcherResponse getTimeAggregatedMetrics(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
// No need to make execute the rest of this if there are no TimeAggregations in the request.
if (entitiesRequest.getTimeAggregationCount() == 0) {
return new EntityFetcherResponse();
}
// Only supported filter is entityIds IN ["id1", "id2", "id3"]
List<String> idColumns = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
String timeColumn = AttributeMetadataUtil.getTimestampAttributeId(attributeMetadataProvider, requestContext, entitiesRequest.getEntityType());
Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
entitiesRequest.getTimeAggregationList().forEach(timeAggregation -> requestContext.mapAliasToTimeAggregation(timeAggregation.getAggregation().getFunction().getAlias(), timeAggregation));
// First group the Aggregations based on the period so that we can issue separate queries
// to QueryService for each different Period.
Collection<List<TimeAggregation>> result = entitiesRequest.getTimeAggregationList().stream().collect(Collectors.groupingBy(TimeAggregation::getPeriod)).values();
Map<EntityKey, Map<String, MetricSeries.Builder>> entityMetricSeriesMap = new LinkedHashMap<>();
for (List<TimeAggregation> batch : result) {
Period period = batch.get(0).getPeriod();
ChronoUnit unit = ChronoUnit.valueOf(period.getUnit());
long periodSecs = Duration.of(period.getValue(), unit).getSeconds();
QueryRequest request = buildTimeSeriesQueryRequest(entitiesRequest, requestContext, periodSecs, batch, idColumns, timeColumn);
if (LOG.isDebugEnabled()) {
LOG.debug("Sending time series queryRequest to query service: ======== \n {}", request.toString());
}
Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(request, requestContext.getHeaders(), requestTimeout);
while (resultSetChunkIterator.hasNext()) {
ResultSetChunk chunk = resultSetChunkIterator.next();
if (LOG.isDebugEnabled()) {
LOG.debug("Received chunk: " + chunk.toString());
}
if (chunk.getRowCount() < 1) {
break;
}
if (!chunk.hasResultSetMetadata()) {
LOG.warn("Chunk doesn't have result metadata so couldn't process the response.");
break;
}
for (Row row : chunk.getRowList()) {
// Construct the entity id from the entityIdAttributeIds columns
EntityKey entityKey = EntityKey.of(IntStream.range(0, idColumns.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
Map<String, MetricSeries.Builder> metricSeriesMap = entityMetricSeriesMap.computeIfAbsent(entityKey, k -> new LinkedHashMap<>());
Interval.Builder intervalBuilder = Interval.newBuilder();
// Second column is the time column
Value value = QueryAndGatewayDtoConverter.convertQueryValueToGatewayValue(row.getColumn(idColumns.size()));
if (value.getValueType() == ValueType.STRING) {
long startTime = Long.parseLong(value.getString());
long endTime = startTime + TimeUnit.SECONDS.toMillis(periodSecs);
intervalBuilder.setStartTimeMillis(startTime);
intervalBuilder.setEndTimeMillis(endTime);
for (int i = idColumns.size() + 1; i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
TimeAggregation timeAggregation = requestContext.getTimeAggregationByAlias(metadata.getColumnName());
if (timeAggregation == null) {
LOG.warn("Couldn't find an aggregate for column: {}", metadata.getColumnName());
continue;
}
FunctionType functionType = timeAggregation.getAggregation().getFunction().getFunction();
AttributeMetadata functionAttributeMetadata = resultKeyToAttributeMetadataMap.get(metadata.getColumnName());
Value convertedValue = QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue(MetricAggregationFunctionUtil.getValueTypeForFunctionType(functionType, functionAttributeMetadata), resultKeyToAttributeMetadataMap, metadata, row.getColumn(i));
List<org.hypertrace.gateway.service.v1.common.Expression> healthExpressions = timeAggregation.getAggregation().getFunction().getArgumentsList().stream().filter(org.hypertrace.gateway.service.v1.common.Expression::hasHealth).collect(Collectors.toList());
Preconditions.checkArgument(healthExpressions.size() <= 1);
Health health = Health.NOT_COMPUTED;
MetricSeries.Builder seriesBuilder = metricSeriesMap.computeIfAbsent(metadata.getColumnName(), k -> getMetricSeriesBuilder(timeAggregation));
seriesBuilder.addValue(Interval.newBuilder(intervalBuilder.build()).setValue(convertedValue).setHealth(health));
}
} else {
LOG.warn("Was expecting STRING values only but received valueType: {}", value.getValueType());
}
}
}
}
Map<EntityKey, Entity.Builder> resultMap = new LinkedHashMap<>();
for (Map.Entry<EntityKey, Map<String, MetricSeries.Builder>> entry : entityMetricSeriesMap.entrySet()) {
Entity.Builder entityBuilder = Entity.newBuilder().setEntityType(entitiesRequest.getEntityType()).setId(entry.getKey().toString()).putAllMetricSeries(entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> getSortedMetricSeries(e.getValue()))));
for (int i = 0; i < idColumns.size(); i++) {
entityBuilder.putAttribute(idColumns.get(i), Value.newBuilder().setString(entry.getKey().getAttributes().get(i)).setValueType(ValueType.STRING).build());
}
resultMap.put(entry.getKey(), entityBuilder);
}
return new EntityFetcherResponse(resultMap);
}
use of org.hypertrace.core.query.service.api.QueryRequest in project gateway-service by hypertrace.
the class BaselineServiceImpl method getBaselineForEntities.
public BaselineEntitiesResponse getBaselineForEntities(String tenantId, BaselineEntitiesRequest originalRequest, Map<String, String> requestHeaders) {
BaselineRequestContext requestContext = getRequestContext(tenantId, requestHeaders, originalRequest);
Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, originalRequest.getEntityType());
baselineEntitiesRequestValidator.validate(originalRequest, attributeMetadataMap);
String timeColumn = AttributeMetadataUtil.getTimestampAttributeId(attributeMetadataProvider, requestContext, originalRequest.getEntityType());
Map<String, BaselineEntity> baselineEntityAggregatedMetricsMap = new HashMap<>();
Map<String, BaselineEntity> baselineEntityTimeSeriesMap = new HashMap<>();
if (originalRequest.getBaselineAggregateRequestCount() > 0) {
// Aggregated Functions data
Period aggTimePeriod = getPeriod(originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis());
long periodSecs = getPeriodInSecs(aggTimePeriod);
long aggStartTime = originalRequest.getStartTimeMillis();
long aggEndTime = originalRequest.getEndTimeMillis();
List<TimeAggregation> timeAggregations = getTimeAggregationsForAggregateExpr(originalRequest, aggStartTime, aggEndTime);
updateAliasMap(requestContext, timeAggregations);
// Take more data to calculate baseline and standard deviation.
long seriesStartTime = getUpdatedStartTime(aggStartTime, aggEndTime);
long seriesEndTime = aggStartTime;
List<String> entityIdAttributes = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, originalRequest.getEntityType());
QueryRequest aggQueryRequest = baselineServiceQueryParser.getQueryRequest(seriesStartTime, seriesEndTime, originalRequest.getEntityIdsList(), timeColumn, timeAggregations, periodSecs, entityIdAttributes);
Iterator<ResultSetChunk> aggResponseChunkIterator = baselineServiceQueryExecutor.executeQuery(requestHeaders, aggQueryRequest);
BaselineEntitiesResponse aggEntitiesResponse = baselineServiceQueryParser.parseQueryResponse(aggResponseChunkIterator, requestContext, entityIdAttributes.size(), originalRequest.getEntityType(), aggStartTime, aggEndTime);
baselineEntityAggregatedMetricsMap = getEntitiesMapFromAggResponse(aggEntitiesResponse);
}
// Time Series data
if (originalRequest.getBaselineMetricSeriesRequestCount() > 0) {
Period timeSeriesPeriod = getTimeSeriesPeriod(originalRequest.getBaselineMetricSeriesRequestList());
long periodSecs = getPeriodInSecs(timeSeriesPeriod);
long alignedStartTime = QueryExpressionUtil.alignToPeriodBoundary(originalRequest.getStartTimeMillis(), periodSecs, true);
long alignedEndTime = QueryExpressionUtil.alignToPeriodBoundary(originalRequest.getEndTimeMillis(), periodSecs, false);
List<TimeAggregation> timeAggregations = getTimeAggregationsForTimeSeriesExpr(originalRequest);
long seriesStartTime = getUpdatedStartTime(alignedStartTime, alignedEndTime);
long seriesEndTime = alignedStartTime;
List<String> entityIdAttributes = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, originalRequest.getEntityType());
QueryRequest timeSeriesQueryRequest = baselineServiceQueryParser.getQueryRequest(seriesStartTime, seriesEndTime, originalRequest.getEntityIdsList(), timeColumn, timeAggregations, periodSecs, entityIdAttributes);
Iterator<ResultSetChunk> timeSeriesChunkIterator = baselineServiceQueryExecutor.executeQuery(requestHeaders, timeSeriesQueryRequest);
BaselineEntitiesResponse timeSeriesEntitiesResponse = baselineServiceQueryParser.parseQueryResponse(timeSeriesChunkIterator, requestContext, entityIdAttributes.size(), originalRequest.getEntityType(), alignedStartTime, alignedEndTime);
baselineEntityTimeSeriesMap = getEntitiesMapFromTimeSeriesResponse(timeSeriesEntitiesResponse, alignedStartTime, alignedEndTime, periodSecs);
}
return mergeEntities(baselineEntityAggregatedMetricsMap, baselineEntityTimeSeriesMap);
}
Aggregations