use of org.hypertrace.gateway.service.entity.EntitiesRequestContext in project gateway-service by hypertrace.
the class EntityDataServiceEntityFetcher method getEntities.
@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
EntityQueryRequest.Builder builder = EntityQueryRequest.newBuilder().setEntityType(entitiesRequest.getEntityType()).setFilter(EntityServiceAndGatewayServiceConverter.convertToEntityServiceFilter(entitiesRequest.getFilter())).addAllSelection(entityIdAttributeIds.stream().map(entityIdAttr -> EntityServiceAndGatewayServiceConverter.createColumnExpression(entityIdAttr).build()).collect(Collectors.toList()));
// add time filter for supported scope
if (!entitiesRequest.getIncludeNonLiveEntities()) {
EntityServiceAndGatewayServiceConverter.addBetweenTimeFilter(entitiesRequest.getStartTimeMillis(), entitiesRequest.getEndTimeMillis(), attributeMetadataProvider, entitiesRequest, builder, requestContext);
}
// Add all expressions in the select that are already not part of the EntityID attributes
entitiesRequest.getSelectionList().stream().filter(ExpressionReader::isAttributeSelection).filter(expression -> ExpressionReader.getAttributeIdFromAttributeSelection(expression).map(attributeId -> !entityIdAttributeIds.contains(attributeId)).orElse(true)).forEach(expression -> builder.addSelection(EntityServiceAndGatewayServiceConverter.convertToEntityServiceExpression(expression)));
int limit = entitiesRequest.getLimit();
if (limit > 0) {
builder.setLimit(limit);
}
int offset = entitiesRequest.getOffset();
if (offset > 0) {
builder.setOffset(offset);
}
if (!entitiesRequest.getOrderByList().isEmpty()) {
builder.addAllOrderBy(EntityServiceAndGatewayServiceConverter.convertToOrderByExpressions(entitiesRequest.getOrderByList()));
}
EntityQueryRequest entityQueryRequest = builder.build();
LOG.debug("Sending Query to EDS ======== \n {}", entityQueryRequest);
Iterator<ResultSetChunk> resultSetChunkIterator = entityQueryServiceClient.execute(builder.build(), requestContext.getHeaders());
Map<String, AttributeMetadata> resultMetadataMap = this.getAttributeMetadataByAlias(requestContext, entitiesRequest);
// We want to retain the order as returned from the respective source. Hence using a
// LinkedHashMap
Map<EntityKey, Builder> entityBuilders = new LinkedHashMap<>();
while (resultSetChunkIterator.hasNext()) {
ResultSetChunk chunk = resultSetChunkIterator.next();
LOG.debug("Received chunk: {}", chunk);
if (chunk.getRowCount() < 1) {
break;
}
for (Row row : chunk.getRowList()) {
// Construct the entity id from the entityIdAttributes columns
EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
entityBuilder.setEntityType(entitiesRequest.getEntityType());
entityBuilder.setId(entityKey.toString());
// as post processing.
for (int i = 0; i < entityIdAttributeIds.size(); i++) {
entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
}
requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
String resultName = chunk.getResultSetMetadata().getColumnMetadata(i).getColumnName();
AttributeMetadata attributeMetadata = resultMetadataMap.get(resultName);
entityBuilder.putAttribute(resultName, EntityServiceAndGatewayServiceConverter.convertQueryValueToGatewayValue(row.getColumn(i), attributeMetadata));
}
}
}
return new EntityFetcherResponse(entityBuilders);
}
use of org.hypertrace.gateway.service.entity.EntitiesRequestContext in project gateway-service by hypertrace.
the class QueryServiceEntityFetcher method getEntities.
@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
// Validate EntitiesRequest
entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
List<org.hypertrace.gateway.service.v1.common.Expression> aggregates = ExpressionReader.getFunctionExpressions(entitiesRequest.getSelectionList());
Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
QueryRequest.Builder builder = constructSelectionQuery(requestContext, entitiesRequest, entityIdAttributeIds, aggregates);
adjustLimitAndOffset(builder, entitiesRequest.getLimit(), entitiesRequest.getOffset());
if (!entitiesRequest.getOrderByList().isEmpty()) {
// Order by from the request.
builder.addAllOrderBy(QueryAndGatewayDtoConverter.convertToQueryOrderByExpressions(entitiesRequest.getOrderByList()));
}
QueryRequest queryRequest = builder.build();
LOG.debug("Sending Query to Query Service ======== \n {}", queryRequest);
Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(queryRequest, requestContext.getHeaders(), requestTimeout);
// We want to retain the order as returned from the respective source. Hence using a
// LinkedHashMap
Map<EntityKey, Entity.Builder> entityBuilders = new LinkedHashMap<>();
while (resultSetChunkIterator.hasNext()) {
ResultSetChunk chunk = resultSetChunkIterator.next();
LOG.debug("Received chunk: {}", chunk);
if (chunk.getRowCount() < 1) {
break;
}
for (Row row : chunk.getRowList()) {
// Construct the entity id from the entityIdAttributeIds columns
EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
entityBuilder.setEntityType(entitiesRequest.getEntityType());
entityBuilder.setId(entityKey.toString());
// as post processing.
for (int i = 0; i < entityIdAttributeIds.size(); i++) {
entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
}
requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
org.hypertrace.core.query.service.api.Value columnValue = row.getColumn(i);
buildEntity(entityBuilder, requestContext, metadata, columnValue, resultKeyToAttributeMetadataMap, aggregates.isEmpty());
}
}
}
return new EntityFetcherResponse(entityBuilders);
}
use of org.hypertrace.gateway.service.entity.EntitiesRequestContext in project gateway-service by hypertrace.
the class ExecutionTreeBuilderTest method test_build_selectAttributesAndFilterWithSameSource_shouldCreateDataFetcherNodeAndPaginateOnlyNode.
@Test
public void test_build_selectAttributesAndFilterWithSameSource_shouldCreateDataFetcherNodeAndPaginateOnlyNode() {
OrderByExpression orderByExpression = buildOrderByExpression(API_STATE_ATTR);
EntitiesRequest entitiesRequest = EntitiesRequest.newBuilder().setEntityType(AttributeScope.API.name()).addSelection(buildExpression(API_STATE_ATTR)).addSelection(buildAggregateExpression(API_NUM_CALLS_ATTR, FunctionType.SUM, "SUM_numCalls", List.of())).setFilter(generateAndOrNotFilter(Operator.AND, generateEQFilter(API_DISCOVERY_STATE, "DISCOVERED"), generateFilter(Operator.GE, API_NUM_CALLS_ATTR, Value.newBuilder().setDouble(60).setValueType(ValueType.DOUBLE).build()))).addOrderBy(orderByExpression).setLimit(10).setOffset(10).build();
EntitiesRequestContext entitiesRequestContext = new EntitiesRequestContext(TENANT_ID, 0L, 10L, "API", "API.startTime", new HashMap<>());
EntityExecutionContext executionContext = new EntityExecutionContext(attributeMetadataProvider, entityIdColumnsConfigs, entitiesRequestContext, entitiesRequest);
ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext);
QueryNode executionTree = executionTreeBuilder.build();
assertNotNull(executionTree);
assertTrue(executionTree instanceof SelectionNode);
assertTrue(((SelectionNode) executionTree).getAggMetricSelectionSources().contains("QS"));
QueryNode paginateOnlyNode = ((SelectionNode) executionTree).getChildNode();
assertTrue(paginateOnlyNode instanceof PaginateOnlyNode);
assertEquals(10, ((PaginateOnlyNode) paginateOnlyNode).getOffset());
assertEquals(10, ((PaginateOnlyNode) paginateOnlyNode).getLimit());
QueryNode dataFetcherNode = ((PaginateOnlyNode) paginateOnlyNode).getChildNode();
assertTrue(dataFetcherNode instanceof DataFetcherNode);
assertEquals("QS", ((DataFetcherNode) dataFetcherNode).getSource());
assertEquals(0, ((DataFetcherNode) dataFetcherNode).getOffset());
assertEquals(20, ((DataFetcherNode) dataFetcherNode).getLimit());
}
use of org.hypertrace.gateway.service.entity.EntitiesRequestContext in project gateway-service by hypertrace.
the class ExecutionTreeBuilderTest method test_build_selectAttributesTimeAggregationAndFilterWithSameSource_shouldCreateDataFetcherNode.
@Test
public void test_build_selectAttributesTimeAggregationAndFilterWithSameSource_shouldCreateDataFetcherNode() {
EntitiesRequest entitiesRequest = EntitiesRequest.newBuilder().setEntityType(AttributeScope.API.name()).addSelection(buildExpression(API_STATE_ATTR)).addSelection(buildAggregateExpression(API_NUM_CALLS_ATTR, FunctionType.SUM, "SUM_numCalls", List.of())).addTimeAggregation(buildTimeAggregation(30, API_NUM_CALLS_ATTR, FunctionType.AVG, "AVG_numCalls", List.of())).setFilter(generateAndOrNotFilter(Operator.AND, generateEQFilter(API_STATE_ATTR, "state1"), generateFilter(Operator.GE, API_NUM_CALLS_ATTR, Value.newBuilder().setDouble(60).setValueType(ValueType.DOUBLE).build()))).setLimit(10).setOffset(0).build();
EntitiesRequestContext entitiesRequestContext = new EntitiesRequestContext(TENANT_ID, 0L, 10L, "API", "API.startTime", new HashMap<>());
EntityExecutionContext executionContext = new EntityExecutionContext(attributeMetadataProvider, entityIdColumnsConfigs, entitiesRequestContext, entitiesRequest);
ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext);
QueryNode executionTree = executionTreeBuilder.build();
assertNotNull(executionTree);
assertTrue(executionTree instanceof SelectionNode);
assertTrue(((SelectionNode) executionTree).getTimeSeriesSelectionSources().contains("QS"));
QueryNode selectionNode = ((SelectionNode) executionTree).getChildNode();
assertTrue(selectionNode instanceof SelectionNode);
assertTrue(((SelectionNode) selectionNode).getAggMetricSelectionSources().contains("QS"));
QueryNode paginateOnlyNode = ((SelectionNode) selectionNode).getChildNode();
assertTrue(paginateOnlyNode instanceof PaginateOnlyNode);
assertEquals(0, ((PaginateOnlyNode) paginateOnlyNode).getOffset());
assertEquals(10, ((PaginateOnlyNode) paginateOnlyNode).getLimit());
QueryNode dataFetcherNode = ((PaginateOnlyNode) paginateOnlyNode).getChildNode();
assertTrue(dataFetcherNode instanceof DataFetcherNode);
assertEquals("QS", ((DataFetcherNode) dataFetcherNode).getSource());
assertEquals(0, ((DataFetcherNode) dataFetcherNode).getOffset());
assertEquals(10, ((DataFetcherNode) dataFetcherNode).getLimit());
}
use of org.hypertrace.gateway.service.entity.EntitiesRequestContext in project gateway-service by hypertrace.
the class ExecutionTreeBuilderTest method nonLiveEntities_noFilters_shouldFetchFromEds.
@Test
public void nonLiveEntities_noFilters_shouldFetchFromEds() {
EntitiesRequest entitiesRequest = EntitiesRequest.newBuilder().setEntityType(AttributeScope.API.name()).addSelection(buildExpression(API_API_ID_ATTR)).setIncludeNonLiveEntities(true).build();
EntitiesRequestContext entitiesRequestContext = new EntitiesRequestContext(TENANT_ID, entitiesRequest.getStartTimeMillis(), entitiesRequest.getEndTimeMillis(), "API", "API.startTime", new HashMap<>());
EntityExecutionContext executionContext = new EntityExecutionContext(attributeMetadataProvider, entityIdColumnsConfigs, entitiesRequestContext, entitiesRequest);
ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext);
QueryNode executionTree = executionTreeBuilder.build();
assertNotNull(executionTree);
assertTrue(executionTree instanceof DataFetcherNode);
// should be EDS, since `setIncludeNonLiveEntities` is set to true, and there are no other
// filters
assertEquals(AttributeSource.EDS.name(), ((DataFetcherNode) executionTree).getSource());
}
Aggregations