Search in sources :

Example 1 with Expression

use of org.hypertrace.gateway.service.v1.common.Expression in project gateway-service by hypertrace.

the class EntityDataServiceEntityFetcher method getEntities.

@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
    EntityQueryRequest.Builder builder = EntityQueryRequest.newBuilder().setEntityType(entitiesRequest.getEntityType()).setFilter(EntityServiceAndGatewayServiceConverter.convertToEntityServiceFilter(entitiesRequest.getFilter())).addAllSelection(entityIdAttributeIds.stream().map(entityIdAttr -> EntityServiceAndGatewayServiceConverter.createColumnExpression(entityIdAttr).build()).collect(Collectors.toList()));
    // add time filter for supported scope
    if (!entitiesRequest.getIncludeNonLiveEntities()) {
        EntityServiceAndGatewayServiceConverter.addBetweenTimeFilter(entitiesRequest.getStartTimeMillis(), entitiesRequest.getEndTimeMillis(), attributeMetadataProvider, entitiesRequest, builder, requestContext);
    }
    // Add all expressions in the select that are already not part of the EntityID attributes
    entitiesRequest.getSelectionList().stream().filter(ExpressionReader::isAttributeSelection).filter(expression -> ExpressionReader.getAttributeIdFromAttributeSelection(expression).map(attributeId -> !entityIdAttributeIds.contains(attributeId)).orElse(true)).forEach(expression -> builder.addSelection(EntityServiceAndGatewayServiceConverter.convertToEntityServiceExpression(expression)));
    int limit = entitiesRequest.getLimit();
    if (limit > 0) {
        builder.setLimit(limit);
    }
    int offset = entitiesRequest.getOffset();
    if (offset > 0) {
        builder.setOffset(offset);
    }
    if (!entitiesRequest.getOrderByList().isEmpty()) {
        builder.addAllOrderBy(EntityServiceAndGatewayServiceConverter.convertToOrderByExpressions(entitiesRequest.getOrderByList()));
    }
    EntityQueryRequest entityQueryRequest = builder.build();
    LOG.debug("Sending Query to EDS  ======== \n {}", entityQueryRequest);
    Iterator<ResultSetChunk> resultSetChunkIterator = entityQueryServiceClient.execute(builder.build(), requestContext.getHeaders());
    Map<String, AttributeMetadata> resultMetadataMap = this.getAttributeMetadataByAlias(requestContext, entitiesRequest);
    // We want to retain the order as returned from the respective source. Hence using a
    // LinkedHashMap
    Map<EntityKey, Builder> entityBuilders = new LinkedHashMap<>();
    while (resultSetChunkIterator.hasNext()) {
        ResultSetChunk chunk = resultSetChunkIterator.next();
        LOG.debug("Received chunk: {}", chunk);
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the entity id from the entityIdAttributes columns
            EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
            entityBuilder.setEntityType(entitiesRequest.getEntityType());
            entityBuilder.setId(entityKey.toString());
            // as post processing.
            for (int i = 0; i < entityIdAttributeIds.size(); i++) {
                entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
            }
            requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
            for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                String resultName = chunk.getResultSetMetadata().getColumnMetadata(i).getColumnName();
                AttributeMetadata attributeMetadata = resultMetadataMap.get(resultName);
                entityBuilder.putAttribute(resultName, EntityServiceAndGatewayServiceConverter.convertQueryValueToGatewayValue(row.getColumn(i), attributeMetadata));
            }
        }
    }
    return new EntityFetcherResponse(entityBuilders);
}
Also used : IntStream(java.util.stream.IntStream) Row(org.hypertrace.entity.query.service.v1.Row) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) EntitiesRequest(org.hypertrace.gateway.service.v1.entity.EntitiesRequest) ResultSetChunk(org.hypertrace.entity.query.service.v1.ResultSetChunk) EntitiesRequestContext(org.hypertrace.gateway.service.entity.EntitiesRequestContext) LoggerFactory(org.slf4j.LoggerFactory) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) ExpressionReader.getExpectedResultNamesForEachAttributeId(org.hypertrace.gateway.service.common.util.ExpressionReader.getExpectedResultNamesForEachAttributeId) ExpressionReader(org.hypertrace.gateway.service.common.util.ExpressionReader) LinkedHashMap(java.util.LinkedHashMap) EntityIdColumnsConfigs(org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs) Map(java.util.Map) TotalEntitiesResponse(org.hypertrace.entity.query.service.v1.TotalEntitiesResponse) EntityQueryServiceClient(org.hypertrace.entity.query.service.client.EntityQueryServiceClient) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) EntityServiceAndGatewayServiceConverter(org.hypertrace.gateway.service.common.converters.EntityServiceAndGatewayServiceConverter) Value(org.hypertrace.gateway.service.v1.common.Value) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) ValueType(org.hypertrace.gateway.service.v1.common.ValueType) Collectors(java.util.stream.Collectors) AttributeMetadataUtil(org.hypertrace.gateway.service.common.util.AttributeMetadataUtil) Entity(org.hypertrace.gateway.service.v1.entity.Entity) List(java.util.List) Entry(java.util.Map.Entry) TotalEntitiesRequest(org.hypertrace.entity.query.service.v1.TotalEntitiesRequest) EntityQueryRequest(org.hypertrace.entity.query.service.v1.EntityQueryRequest) AttributeMetadataProvider(org.hypertrace.gateway.service.common.AttributeMetadataProvider) Builder(org.hypertrace.gateway.service.v1.entity.Entity.Builder) LinkedHashMap(java.util.LinkedHashMap) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) List(java.util.List) Row(org.hypertrace.entity.query.service.v1.Row) ExpressionReader(org.hypertrace.gateway.service.common.util.ExpressionReader) EntityQueryRequest(org.hypertrace.entity.query.service.v1.EntityQueryRequest) ResultSetChunk(org.hypertrace.entity.query.service.v1.ResultSetChunk)

Example 2 with Expression

use of org.hypertrace.gateway.service.v1.common.Expression in project gateway-service by hypertrace.

the class EntityInteractionsFetcher method buildQueryRequests.

@VisibleForTesting
Map<String, QueryRequest> buildQueryRequests(long startTime, long endTime, String spaceId, String entityType, InteractionsRequest interactionsRequest, Set<EntityKey> entityIds, boolean incoming, RequestContext requestContext) {
    Set<String> entityTypes = getOtherEntityTypes(interactionsRequest.getFilter());
    if (entityTypes.isEmpty()) {
        return Collections.emptyMap();
    }
    QueryRequest.Builder builder = QueryRequest.newBuilder();
    // Filter should include the timestamp filters from parent request first
    Filter.Builder filterBuilder = Filter.newBuilder().setOperator(Operator.AND).addChildFilter(QueryRequestUtil.createBetweenTimesFilter(AttributeMetadataUtil.getTimestampAttributeId(metadataProvider, requestContext, SCOPE), startTime, endTime));
    this.buildSpaceQueryFilterIfNeeded(requestContext, spaceId).ifPresent(filterBuilder::addChildFilter);
    List<String> idColumns = getEntityIdColumnsFromInteraction(DomainEntityType.valueOf(entityType), !incoming);
    // Add a filter on the entityIds
    filterBuilder.addChildFilter(createFilterForEntityKeys(idColumns, entityIds));
    // Group by the entity id column first, then the other end entity type for the interaction.
    List<org.hypertrace.core.query.service.api.Expression> idExpressions = idColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList());
    builder.addAllGroupBy(idExpressions);
    List<org.hypertrace.core.query.service.api.Expression> selections = new ArrayList<>();
    for (Expression expression : interactionsRequest.getSelectionList()) {
        // Ignore the predefined selections because they're handled specially.
        if (ExpressionReader.isSimpleAttributeSelection(expression) && SELECTIONS_TO_IGNORE.contains(ExpressionReader.getAttributeIdFromAttributeSelection(expression).orElseThrow())) {
            continue;
        }
        // Selection should have metrics and attributes that were requested
        selections.add(QueryAndGatewayDtoConverter.convertToQueryExpression(expression).build());
    }
    // so we add count(*) as a dummy placeholder if there are no explicit selectors.
    if (selections.isEmpty()) {
        selections.add(QueryRequestUtil.createCountByColumnSelection(Optional.ofNullable(idColumns.get(0)).orElseThrow()));
    }
    QueryRequest protoType = builder.build();
    Filter protoTypeFilter = filterBuilder.build();
    Map<String, QueryRequest> queryRequests = new HashMap<>();
    // response time.
    for (String e : entityTypes) {
        DomainEntityType otherEntityType = DomainEntityType.valueOf(e.toUpperCase());
        // Get the filters from the interactions request to 'AND' them with the timestamp filter.
        Filter.Builder filterCopy = Filter.newBuilder(protoTypeFilter);
        filterCopy.addChildFilter(convertToQueryFilter(interactionsRequest.getFilter(), otherEntityType));
        QueryRequest.Builder builderCopy = QueryRequest.newBuilder(protoType);
        builderCopy.setFilter(filterCopy);
        List<String> otherEntityIdColumns = getEntityIdColumnsFromInteraction(otherEntityType, incoming);
        List<org.hypertrace.core.query.service.api.Expression> otherIdExpressions = otherEntityIdColumns.stream().map(QueryRequestUtil::createAttributeExpression).collect(Collectors.toList());
        builderCopy.addAllGroupBy(otherIdExpressions);
        // Add all selections in the correct order. First id, then other entity id and finally
        // the remaining selections.
        builderCopy.addAllSelection(idExpressions);
        builderCopy.addAllSelection(otherIdExpressions);
        selections.forEach(builderCopy::addSelection);
        int limit = interactionsRequest.getLimit();
        if (limit > 0) {
            builderCopy.setLimit(limit);
        } else {
            builderCopy.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT);
        }
        queryRequests.put(e, builderCopy.build());
    }
    return queryRequests;
}
Also used : QueryRequest(org.hypertrace.core.query.service.api.QueryRequest) DomainEntityType(org.hypertrace.gateway.service.v1.common.DomainEntityType) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Filter(org.hypertrace.core.query.service.api.Filter) QueryRequestUtil.createFilter(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createFilter) Expression(org.hypertrace.gateway.service.v1.common.Expression) QueryRequestUtil.createStringArrayLiteralExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringArrayLiteralExpression) QueryRequestUtil.createStringNullLiteralExpression(org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringNullLiteralExpression) FunctionExpression(org.hypertrace.gateway.service.v1.common.FunctionExpression) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 3 with Expression

use of org.hypertrace.gateway.service.v1.common.Expression in project gateway-service by hypertrace.

the class EntityInteractionsFetcher method getEntityKeyValues.

private Set<EntityKey> getEntityKeyValues(Expression expression) {
    Preconditions.checkArgument(expression.getValueCase() == ValueCase.LITERAL);
    Value value = expression.getLiteral().getValue();
    if (value.getValueType() == ValueType.STRING) {
        return Collections.singleton(EntityKey.from(value.getString()));
    }
    if (value.getValueType() == ValueType.STRING_ARRAY) {
        return value.getStringArrayList().stream().map(EntityKey::from).collect(Collectors.toSet());
    }
    throw new IllegalArgumentException("Expected STRING value but received unhandled type: " + value.getValueType());
}
Also used : Value(org.hypertrace.gateway.service.v1.common.Value) AggregatedMetricValue(org.hypertrace.gateway.service.v1.common.AggregatedMetricValue)

Example 4 with Expression

use of org.hypertrace.gateway.service.v1.common.Expression in project gateway-service by hypertrace.

the class EntityInteractionsFetcher method parseResultSet.

private void parseResultSet(String entityType, String otherEntityType, Collection<Expression> selections, Map<String, FunctionExpression> metricToAggFunction, Iterator<ResultSetChunk> resultset, boolean incoming, Map<EntityKey, Builder> entityIdToBuilders, RequestContext requestContext) {
    Map<String, AttributeMetadata> attributeMetadataMap = metadataProvider.getAttributesMetadata(requestContext, SCOPE);
    Map<String, AttributeKind> aliasToAttributeKind = MetricAggregationFunctionUtil.getValueTypeForFunctionType(metricToAggFunction, attributeMetadataMap);
    while (resultset.hasNext()) {
        ResultSetChunk chunk = resultset.next();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Received chunk: " + chunk.toString());
        }
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the from/to EntityKeys from the columns
            List<String> idColumns = getEntityIdColumnsFromInteraction(DomainEntityType.valueOf(entityType.toUpperCase()), // Note: We add the selections it in this order
            !incoming);
            EntityKey entityId = EntityKey.of(IntStream.range(0, idColumns.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            List<String> otherIdColumns = getEntityIdColumnsFromInteraction(DomainEntityType.valueOf(otherEntityType.toUpperCase()), incoming);
            EntityKey otherEntityId = EntityKey.of(IntStream.range(idColumns.size(), idColumns.size() + otherIdColumns.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            EntityInteraction.Builder interaction = EntityInteraction.newBuilder();
            addInteractionEdges(interaction, selections, incoming ? otherEntityType : entityType, incoming ? otherEntityId : entityId, incoming ? entityType : otherEntityType, incoming ? entityId : otherEntityId);
            for (int i = idColumns.size() + otherIdColumns.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
                // Ignore the count column since we introduced that ourselves into the query.
                if (StringUtils.equalsIgnoreCase(COUNT_COLUMN_NAME, metadata.getColumnName())) {
                    continue;
                }
                // Check if this is an attribute vs metric and set it accordingly on the interaction.
                if (metricToAggFunction.containsKey(metadata.getColumnName())) {
                    Value value = QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue(aliasToAttributeKind, attributeMetadataMap, metadata, row.getColumn(i));
                    interaction.putMetrics(metadata.getColumnName(), AggregatedMetricValue.newBuilder().setValue(value).setFunction(metricToAggFunction.get(metadata.getColumnName()).getFunction()).build());
                } else {
                    interaction.putAttribute(metadata.getColumnName(), QueryAndGatewayDtoConverter.convertQueryValueToGatewayValue(row.getColumn(i), attributeMetadataMap.get(metadata.getColumnName())));
                }
            }
            if (incoming) {
                entityIdToBuilders.get(entityId).addIncomingInteraction(interaction);
            } else {
                entityIdToBuilders.get(entityId).addOutgoingInteraction(interaction);
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug(interaction.build().toString());
            }
        }
    }
}
Also used : ColumnMetadata(org.hypertrace.core.query.service.api.ColumnMetadata) AttributeKind(org.hypertrace.core.attribute.service.v1.AttributeKind) EntityInteraction(org.hypertrace.gateway.service.v1.entity.EntityInteraction) EntityKey(org.hypertrace.gateway.service.entity.EntityKey) AttributeMetadata(org.hypertrace.core.attribute.service.v1.AttributeMetadata) Value(org.hypertrace.gateway.service.v1.common.Value) AggregatedMetricValue(org.hypertrace.gateway.service.v1.common.AggregatedMetricValue) Row(org.hypertrace.core.query.service.api.Row) ResultSetChunk(org.hypertrace.core.query.service.api.ResultSetChunk)

Example 5 with Expression

use of org.hypertrace.gateway.service.v1.common.Expression in project gateway-service by hypertrace.

the class BaselineServiceImpl method getTimeAggregationsForTimeSeriesExpr.

private List<TimeAggregation> getTimeAggregationsForTimeSeriesExpr(BaselineEntitiesRequest originalRequest) {
    List<BaselineTimeAggregation> timeSeriesList = originalRequest.getBaselineMetricSeriesRequestList();
    List<TimeAggregation> timeAggregations = new ArrayList<>();
    for (BaselineTimeAggregation timeAggregation : timeSeriesList) {
        Expression aggregation = Expression.newBuilder().setFunction(timeAggregation.getAggregation()).build();
        TimeAggregation agg = TimeAggregation.newBuilder().setAggregation(aggregation).setPeriod(timeAggregation.getPeriod()).build();
        timeAggregations.add(agg);
    }
    return timeAggregations;
}
Also used : Expression(org.hypertrace.gateway.service.v1.common.Expression) FunctionExpression(org.hypertrace.gateway.service.v1.common.FunctionExpression) ArrayList(java.util.ArrayList) TimeAggregation(org.hypertrace.gateway.service.v1.common.TimeAggregation) BaselineTimeAggregation(org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation) BaselineTimeAggregation(org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation)

Aggregations

Expression (org.hypertrace.gateway.service.v1.common.Expression)31 Test (org.junit.jupiter.api.Test)27 OrderByExpression (org.hypertrace.gateway.service.v1.common.OrderByExpression)18 EntitiesRequest (org.hypertrace.gateway.service.v1.entity.EntitiesRequest)16 EntitiesRequestAndResponseUtils.buildExpression (org.hypertrace.gateway.service.common.EntitiesRequestAndResponseUtils.buildExpression)11 FunctionExpression (org.hypertrace.gateway.service.v1.common.FunctionExpression)11 AttributeMetadata (org.hypertrace.core.attribute.service.v1.AttributeMetadata)10 HashMap (java.util.HashMap)9 EntityFetcherResponse (org.hypertrace.gateway.service.common.datafetcher.EntityFetcherResponse)9 EntitiesRequestContext (org.hypertrace.gateway.service.entity.EntitiesRequestContext)9 EntityKey (org.hypertrace.gateway.service.entity.EntityKey)9 ArrayList (java.util.ArrayList)8 EntitiesRequestAndResponseUtils.buildAggregateExpression (org.hypertrace.gateway.service.common.EntitiesRequestAndResponseUtils.buildAggregateExpression)8 EntitiesRequestAndResponseUtils.buildOrderByExpression (org.hypertrace.gateway.service.common.EntitiesRequestAndResponseUtils.buildOrderByExpression)8 Builder (org.hypertrace.gateway.service.v1.entity.Entity.Builder)8 List (java.util.List)7 EntityResponse (org.hypertrace.gateway.service.common.datafetcher.EntityResponse)7 Filter (org.hypertrace.gateway.service.v1.common.Filter)7 TimeAggregation (org.hypertrace.gateway.service.v1.common.TimeAggregation)7 Map (java.util.Map)6