Search in sources :

Example 1 with EntityIdColumnsConfigs

use of org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs in project gateway-service by hypertrace.

the class EntityDataServiceEntityFetcher method getEntities.

@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
    EntityQueryRequest.Builder builder = EntityQueryRequest.newBuilder().setEntityType(entitiesRequest.getEntityType()).setFilter(EntityServiceAndGatewayServiceConverter.convertToEntityServiceFilter(entitiesRequest.getFilter())).addAllSelection(entityIdAttributeIds.stream().map(entityIdAttr -> EntityServiceAndGatewayServiceConverter.createColumnExpression(entityIdAttr).build()).collect(Collectors.toList()));
    // add time filter for supported scope
    if (!entitiesRequest.getIncludeNonLiveEntities()) {
        EntityServiceAndGatewayServiceConverter.addBetweenTimeFilter(entitiesRequest.getStartTimeMillis(), entitiesRequest.getEndTimeMillis(), attributeMetadataProvider, entitiesRequest, builder, requestContext);
    }
    // Add all expressions in the select that are already not part of the EntityID attributes
    entitiesRequest.getSelectionList().stream().filter(ExpressionReader::isAttributeSelection).filter(expression -> ExpressionReader.getAttributeIdFromAttributeSelection(expression).map(attributeId -> !entityIdAttributeIds.contains(attributeId)).orElse(true)).forEach(expression -> builder.addSelection(EntityServiceAndGatewayServiceConverter.convertToEntityServiceExpression(expression)));
    int limit = entitiesRequest.getLimit();
    if (limit > 0) {
        builder.setLimit(limit);
    }
    int offset = entitiesRequest.getOffset();
    if (offset > 0) {
        builder.setOffset(offset);
    }
    if (!entitiesRequest.getOrderByList().isEmpty()) {
        builder.addAllOrderBy(EntityServiceAndGatewayServiceConverter.convertToOrderByExpressions(entitiesRequest.getOrderByList()));
    }
    EntityQueryRequest entityQueryRequest = builder.build();
    LOG.debug("Sending Query to EDS  ======== \n {}", entityQueryRequest);
    Iterator<ResultSetChunk> resultSetChunkIterator = entityQueryServiceClient.execute(builder.build(), requestContext.getHeaders());
    Map<String, AttributeMetadata> resultMetadataMap = this.getAttributeMetadataByAlias(requestContext, entitiesRequest);
    // We want to retain the order as returned from the respective source. Hence using a
    // LinkedHashMap
    Map<EntityKey, Builder> entityBuilders = new LinkedHashMap<>();
    while (resultSetChunkIterator.hasNext()) {
        ResultSetChunk chunk = resultSetChunkIterator.next();
        LOG.debug("Received chunk: {}", chunk);
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the entity id from the entityIdAttributes columns
            EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
            entityBuilder.setEntityType(entitiesRequest.getEntityType());
            entityBuilder.setId(entityKey.toString());
            // as post processing.
            for (int i = 0; i < entityIdAttributeIds.size(); i++) {
                entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
            }
            requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
            for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                String resultName = chunk.getResultSetMetadata().getColumnMetadata(i).getColumnName();
                AttributeMetadata attributeMetadata = resultMetadataMap.get(resultName);
                entityBuilder.putAttribute(resultName, EntityServiceAndGatewayServiceConverter.convertQueryValueToGatewayValue(row.getColumn(i), attributeMetadata));
            }
        }
    }
    return new EntityFetcherResponse(entityBuilders);
}
Also used : IntStream(java.util.stream.IntStream) Row(org.hypertrace.entity.query.service.v1.Row) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) EntitiesRequest(org.hypertrace.gateway.service.v1.entity.EntitiesRequest) ResultSetChunk(org.hypertrace.entity.query.service.v1.ResultSetChunk) EntitiesRequestContext(org.hypertrace.gateway.service.entity.EntitiesRequestContext) LoggerFactory(org.slf4j.LoggerFactory) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) ExpressionReader.getExpectedResultNamesForEachAttributeId(org.hypertrace.gateway.service.common.util.ExpressionReader.getExpectedResultNamesForEachAttributeId) ExpressionReader(org.hypertrace.gateway.service.common.util.ExpressionReader) LinkedHashMap(java.util.LinkedHashMap) EntityIdColumnsConfigs(org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs) Map(java.util.Map) TotalEntitiesResponse(org.hypertrace.entity.query.service.v1.TotalEntitiesResponse) EntityQueryServiceClient(org.hypertrace.entity.query.service.client.EntityQueryServiceClient) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) EntityServiceAndGatewayServiceConverter(org.hypertrace.gateway.service.common.converters.EntityServiceAndGatewayServiceConverter) Value(org.hypertrace.gateway.service.v1.common.Value) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) ValueType(org.hypertrace.gateway.service.v1.common.ValueType) Collectors(java.util.stream.Collectors) AttributeMetadataUtil(org.hypertrace.gateway.service.common.util.AttributeMetadataUtil) Entity(org.hypertrace.gateway.service.v1.entity.Entity) List(java.util.List) Entry(java.util.Map.Entry) TotalEntitiesRequest(org.hypertrace.entity.query.service.v1.TotalEntitiesRequest) EntityQueryRequest(org.hypertrace.entity.query.service.v1.EntityQueryRequest) AttributeMetadataProvider(org.hypertrace.gateway.service.common.AttributeMetadataProvider) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) LinkedHashMap(java.util.LinkedHashMap) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) List(java.util.List) Row(org.hypertrace.entity.query.service.v1.Row) ExpressionReader(org.hypertrace.gateway.service.common.util.ExpressionReader) EntityQueryRequest(org.hypertrace.entity.query.service.v1.EntityQueryRequest) ResultSetChunk(org.hypertrace.entity.query.service.v1.ResultSetChunk)

Example 2 with EntityIdColumnsConfigs

use of org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs in project gateway-service by hypertrace.

the class QueryServiceEntityFetcher method getEntities.

@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
    Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
    // Validate EntitiesRequest
    entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
    List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    List<org.hypertrace.gateway.service.v1.common.Expression> aggregates = ExpressionReader.getFunctionExpressions(entitiesRequest.getSelectionList());
    Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
    QueryRequest.Builder builder = constructSelectionQuery(requestContext, entitiesRequest, entityIdAttributeIds, aggregates);
    adjustLimitAndOffset(builder, entitiesRequest.getLimit(), entitiesRequest.getOffset());
    if (!entitiesRequest.getOrderByList().isEmpty()) {
        // Order by from the request.
        builder.addAllOrderBy(QueryAndGatewayDtoConverter.convertToQueryOrderByExpressions(entitiesRequest.getOrderByList()));
    }
    QueryRequest queryRequest = builder.build();
    LOG.debug("Sending Query to Query Service ======== \n {}", queryRequest);
    Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(queryRequest, requestContext.getHeaders(), requestTimeout);
    // We want to retain the order as returned from the respective source. Hence using a
    // LinkedHashMap
    Map<EntityKey, Entity.Builder> entityBuilders = new LinkedHashMap<>();
    while (resultSetChunkIterator.hasNext()) {
        ResultSetChunk chunk = resultSetChunkIterator.next();
        LOG.debug("Received chunk: {}", chunk);
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the entity id from the entityIdAttributeIds columns
            EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
            entityBuilder.setEntityType(entitiesRequest.getEntityType());
            entityBuilder.setId(entityKey.toString());
            // as post processing.
            for (int i = 0; i < entityIdAttributeIds.size(); i++) {
                entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
            }
            requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
            for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
                org.hypertrace.core.query.service.api.Value columnValue = row.getColumn(i);
                buildEntity(entityBuilder, requestContext, metadata, columnValue, resultKeyToAttributeMetadataMap, aggregates.isEmpty());
            }
        }
    }
    return new EntityFetcherResponse(entityBuilders);
}
Also used : LoggerFactory(org.slf4j.LoggerFactory) QueryAndGatewayDtoConverter(org.hypertrace.gateway.service.common.converters.QueryAndGatewayDtoConverter) StringUtils(org.apache.commons.lang3.StringUtils) MetricSeries(org.hypertrace.gateway.service.v1.common.MetricSeries) QueryRequestUtil.createTimeColumnGroupByExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createTimeColumnGroupByExpression) EntityIdColumnsConfigs(org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs) Duration(java.time.Duration) Map(java.util.Map) Objects.isNull(java.util.Objects.isNull) Operator(org.hypertrace.core.query.service.api.Operator) Row(org.hypertrace.core.query.service.api.Row) Filter(org.hypertrace.core.query.service.api.Filter) QueryAndGatewayDtoConverter.convertToQueryExpression(org.hypertrace.gateway.service.common.converters.QueryAndGatewayDtoConverter.convertToQueryExpression) Collection(java.util.Collection) Value(org.hypertrace.gateway.service.v1.common.Value) ColumnMetadata(org.hypertrace.core.query.service.api.ColumnMetadata) Interval(org.hypertrace.gateway.service.v1.common.Interval) Streams(com.google.common.collect.Streams) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) ValueType(org.hypertrace.gateway.service.v1.common.ValueType) Collectors(java.util.stream.Collectors) List(java.util.List) QueryRequest(org.hypertrace.core.query.service.api.QueryRequest) QueryRequestUtil(org.hypertrace.gateway.service.common.converters.QueryRequestUtil) Optional(java.util.Optional) ResultSetChunk(org.hypertrace.core.query.service.api.ResultSetChunk) AttributeMetadataProvider(org.hypertrace.gateway.service.common.AttributeMetadataProvider) IntStream(java.util.stream.IntStream) QueryExpressionUtil(org.hypertrace.gateway.service.common.util.QueryExpressionUtil) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) EntitiesRequest(org.hypertrace.gateway.service.v1.entity.EntitiesRequest) EntitiesRequestValidator(org.hypertrace.gateway.service.entity.EntitiesRequestValidator) EntitiesRequestContext(org.hypertrace.gateway.service.entity.EntitiesRequestContext) AggregatedMetricValue(org.hypertrace.gateway.service.v1.common.AggregatedMetricValue) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) ExpressionReader.getExpectedResultNamesForEachAttributeId(org.hypertrace.gateway.service.common.util.ExpressionReader.getExpectedResultNamesForEachAttributeId) ExpressionReader(org.hypertrace.gateway.service.common.util.ExpressionReader) LinkedHashMap(java.util.LinkedHashMap) Period(org.hypertrace.gateway.service.v1.common.Period) QueryRequestContext(org.hypertrace.gateway.service.common.QueryRequestContext) QueryRequestUtil.createCountByColumnSelection(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createCountByColumnSelection) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Health(org.hypertrace.gateway.service.v1.common.Health) QueryRequestUtil.createStringNullLiteralExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringNullLiteralExpression) TimeAggregation(org.hypertrace.gateway.service.v1.common.TimeAggregation) MetricAggregationFunctionUtil(org.hypertrace.gateway.service.common.util.MetricAggregationFunctionUtil) QueryServiceClient(org.hypertrace.core.query.service.client.QueryServiceClient) AttributeMetadataUtil(org.hypertrace.gateway.service.common.util.AttributeMetadataUtil) TimeUnit(java.util.concurrent.TimeUnit) Entity(org.hypertrace.gateway.service.v1.entity.Entity) ChronoUnit(java.time.temporal.ChronoUnit) Expression(org.hypertrace.core.query.service.api.Expression) FunctionExpression(org.hypertrace.gateway.service.v1.common.FunctionExpression) QueryRequestUtil.createDistinctCountByColumnSelection(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createDistinctCountByColumnSelection) Preconditions(com.google.common.base.Preconditions) FunctionType(org.hypertrace.gateway.service.v1.common.FunctionType) QueryRequestUtil.createFilter(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createFilter) ColumnMetadata(org.hypertrace.core.query.service.api.ColumnMetadata) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) LinkedHashMap(java.util.LinkedHashMap) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) List(java.util.List) QueryRequest(org.hypertrace.core.query.service.api.QueryRequest) QueryRequestUtil.createTimeColumnGroupByExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createTimeColumnGroupByExpression) QueryAndGatewayDtoConverter.convertToQueryExpression(org.hypertrace.gateway.service.common.converters.QueryAndGatewayDtoConverter.convertToQueryExpression) QueryRequestUtil.createStringNullLiteralExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringNullLiteralExpression) Expression(org.hypertrace.core.query.service.api.Expression) FunctionExpression(org.hypertrace.gateway.service.v1.common.FunctionExpression) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) Row(org.hypertrace.core.query.service.api.Row) ResultSetChunk(org.hypertrace.core.query.service.api.ResultSetChunk)

Example 3 with EntityIdColumnsConfigs

use of org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs in project gateway-service by hypertrace.

the class ExecutionContextTest method setup.

@BeforeEach
public void setup() {
    attributeMetadataProvider = mock(AttributeMetadataProvider.class);
    entityIdColumnsConfigs = mock(EntityIdColumnsConfigs.class);
    when(attributeMetadataProvider.getAttributesMetadata(any(RequestContext.class), eq(AttributeScope.API.name()))).thenReturn(attributeSources);
    attributeSources.forEach((attributeId, attribute) -> when(attributeMetadataProvider.getAttributeMetadata(any(RequestContext.class), eq(attribute.getScopeString()), eq(attribute.getKey()))).thenReturn(Optional.of(attribute)));
    entitiesRequestContext = new EntitiesRequestContext(TENANT_ID, 0, 100, "API", "API.startTime", new HashMap<>());
}
Also used : HashMap(java.util.HashMap) EntitiesRequestContext(org.hypertrace.gateway.service.entity.EntitiesRequestContext) RequestContext(org.hypertrace.gateway.service.common.RequestContext) EntitiesRequestContext(org.hypertrace.gateway.service.entity.EntitiesRequestContext) EntityIdColumnsConfigs(org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs) AttributeMetadataProvider(org.hypertrace.gateway.service.common.AttributeMetadataProvider) BeforeEach(org.junit.jupiter.api.BeforeEach)

Aggregations

AttributeMetadataProvider (org.hypertrace.gateway.service.common.AttributeMetadataProvider)3 EntitiesRequestContext (org.hypertrace.gateway.service.entity.EntitiesRequestContext)3 EntityIdColumnsConfigs (org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs)3 Iterator (java.util.Iterator)2 LinkedHashMap (java.util.LinkedHashMap)2 List (java.util.List)2 Map (java.util.Map)2 Collectors (java.util.stream.Collectors)2 IntStream (java.util.stream.IntStream)2 AttributeMetadata (org.hypertrace.core.attribute.service.v1.AttributeMetadata)2 AttributeMetadataUtil (org.hypertrace.gateway.service.common.util.AttributeMetadataUtil)2 ExpressionReader (org.hypertrace.gateway.service.common.util.ExpressionReader)2 ExpressionReader.getExpectedResultNamesForEachAttributeId (org.hypertrace.gateway.service.common.util.ExpressionReader.getExpectedResultNamesForEachAttributeId)2 EntityKey (org.hypertrace.gateway.service.entity.EntityKey)2 Value (org.hypertrace.gateway.service.v1.common.Value)2 ValueType (org.hypertrace.gateway.service.v1.common.ValueType)2 EntitiesRequest (org.hypertrace.gateway.service.v1.entity.EntitiesRequest)2 Entity (org.hypertrace.gateway.service.v1.entity.Entity)2 Builder (org.hypertrace.gateway.service.v1.entity.Entity.Builder)2 Logger (org.slf4j.Logger)2