Examples with AttributeMetadataProvider org.hypertrace.gateway.service.common.AttributeMetadataProvider used on opensource projects

Example 1 with AttributeMetadataProvider

use of org.hypertrace.gateway.service.common.AttributeMetadataProvider in project gateway-service by hypertrace.

the class EntityDataServiceEntityFetcher method getEntities.

@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
    EntityQueryRequest.Builder builder = EntityQueryRequest.newBuilder().setEntityType(entitiesRequest.getEntityType()).setFilter(EntityServiceAndGatewayServiceConverter.convertToEntityServiceFilter(entitiesRequest.getFilter())).addAllSelection(entityIdAttributeIds.stream().map(entityIdAttr -> EntityServiceAndGatewayServiceConverter.createColumnExpression(entityIdAttr).build()).collect(Collectors.toList()));
    // add time filter for supported scope
    if (!entitiesRequest.getIncludeNonLiveEntities()) {
        EntityServiceAndGatewayServiceConverter.addBetweenTimeFilter(entitiesRequest.getStartTimeMillis(), entitiesRequest.getEndTimeMillis(), attributeMetadataProvider, entitiesRequest, builder, requestContext);
    }
    // Add all expressions in the select that are already not part of the EntityID attributes
    entitiesRequest.getSelectionList().stream().filter(ExpressionReader::isAttributeSelection).filter(expression -> ExpressionReader.getAttributeIdFromAttributeSelection(expression).map(attributeId -> !entityIdAttributeIds.contains(attributeId)).orElse(true)).forEach(expression -> builder.addSelection(EntityServiceAndGatewayServiceConverter.convertToEntityServiceExpression(expression)));
    int limit = entitiesRequest.getLimit();
    if (limit > 0) {
        builder.setLimit(limit);
    }
    int offset = entitiesRequest.getOffset();
    if (offset > 0) {
        builder.setOffset(offset);
    }
    if (!entitiesRequest.getOrderByList().isEmpty()) {
        builder.addAllOrderBy(EntityServiceAndGatewayServiceConverter.convertToOrderByExpressions(entitiesRequest.getOrderByList()));
    }
    EntityQueryRequest entityQueryRequest = builder.build();
    LOG.debug("Sending Query to EDS  ======== \n {}", entityQueryRequest);
    Iterator<ResultSetChunk> resultSetChunkIterator = entityQueryServiceClient.execute(builder.build(), requestContext.getHeaders());
    Map<String, AttributeMetadata> resultMetadataMap = this.getAttributeMetadataByAlias(requestContext, entitiesRequest);
    // We want to retain the order as returned from the respective source. Hence using a
    // LinkedHashMap
    Map<EntityKey, Builder> entityBuilders = new LinkedHashMap<>();
    while (resultSetChunkIterator.hasNext()) {
        ResultSetChunk chunk = resultSetChunkIterator.next();
        LOG.debug("Received chunk: {}", chunk);
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the entity id from the entityIdAttributes columns
            EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
            entityBuilder.setEntityType(entitiesRequest.getEntityType());
            entityBuilder.setId(entityKey.toString());
            // as post processing.
            for (int i = 0; i < entityIdAttributeIds.size(); i++) {
                entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
            }
            requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
            for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                String resultName = chunk.getResultSetMetadata().getColumnMetadata(i).getColumnName();
                AttributeMetadata attributeMetadata = resultMetadataMap.get(resultName);
                entityBuilder.putAttribute(resultName, EntityServiceAndGatewayServiceConverter.convertQueryValueToGatewayValue(row.getColumn(i), attributeMetadata));
            }
        }
    }
    return new EntityFetcherResponse(entityBuilders);
}

Example 2 with AttributeMetadataProvider

use of org.hypertrace.gateway.service.common.AttributeMetadataProvider in project gateway-service by hypertrace.

the class QueryServiceEntityFetcher method getEntities.

@Override
public EntityFetcherResponse getEntities(EntitiesRequestContext requestContext, EntitiesRequest entitiesRequest) {
    Map<String, AttributeMetadata> attributeMetadataMap = attributeMetadataProvider.getAttributesMetadata(requestContext, entitiesRequest.getEntityType());
    Map<String, AttributeMetadata> resultKeyToAttributeMetadataMap = this.remapAttributeMetadataByResultName(entitiesRequest, attributeMetadataMap);
    // Validate EntitiesRequest
    entitiesRequestValidator.validate(entitiesRequest, attributeMetadataMap);
    List<String> entityIdAttributeIds = AttributeMetadataUtil.getIdAttributeIds(attributeMetadataProvider, entityIdColumnsConfigs, requestContext, entitiesRequest.getEntityType());
    List<org.hypertrace.gateway.service.v1.common.Expression> aggregates = ExpressionReader.getFunctionExpressions(entitiesRequest.getSelectionList());
    Map<String, List<String>> requestedAliasesByEntityIdAttributeIds = getExpectedResultNamesForEachAttributeId(entitiesRequest.getSelectionList(), entityIdAttributeIds);
    QueryRequest.Builder builder = constructSelectionQuery(requestContext, entitiesRequest, entityIdAttributeIds, aggregates);
    adjustLimitAndOffset(builder, entitiesRequest.getLimit(), entitiesRequest.getOffset());
    if (!entitiesRequest.getOrderByList().isEmpty()) {
        // Order by from the request.
        builder.addAllOrderBy(QueryAndGatewayDtoConverter.convertToQueryOrderByExpressions(entitiesRequest.getOrderByList()));
    }
    QueryRequest queryRequest = builder.build();
    LOG.debug("Sending Query to Query Service ======== \n {}", queryRequest);
    Iterator<ResultSetChunk> resultSetChunkIterator = queryServiceClient.executeQuery(queryRequest, requestContext.getHeaders(), requestTimeout);
    // We want to retain the order as returned from the respective source. Hence using a
    // LinkedHashMap
    Map<EntityKey, Entity.Builder> entityBuilders = new LinkedHashMap<>();
    while (resultSetChunkIterator.hasNext()) {
        ResultSetChunk chunk = resultSetChunkIterator.next();
        LOG.debug("Received chunk: {}", chunk);
        if (chunk.getRowCount() < 1) {
            break;
        }
        for (Row row : chunk.getRowList()) {
            // Construct the entity id from the entityIdAttributeIds columns
            EntityKey entityKey = EntityKey.of(IntStream.range(0, entityIdAttributeIds.size()).mapToObj(value -> row.getColumn(value).getString()).toArray(String[]::new));
            Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder());
            entityBuilder.setEntityType(entitiesRequest.getEntityType());
            entityBuilder.setId(entityKey.toString());
            // as post processing.
            for (int i = 0; i < entityIdAttributeIds.size(); i++) {
                entityBuilder.putAttribute(entityIdAttributeIds.get(i), Value.newBuilder().setString(entityKey.getAttributes().get(i)).setValueType(ValueType.STRING).build());
            }
            requestedAliasesByEntityIdAttributeIds.forEach((attributeId, requestedAliasList) -> requestedAliasList.forEach(requestedAlias -> entityBuilder.putAttribute(requestedAlias, entityBuilder.getAttributeOrThrow(attributeId))));
            for (int i = entityIdAttributeIds.size(); i < chunk.getResultSetMetadata().getColumnMetadataCount(); i++) {
                ColumnMetadata metadata = chunk.getResultSetMetadata().getColumnMetadata(i);
                org.hypertrace.core.query.service.api.Value columnValue = row.getColumn(i);
                buildEntity(entityBuilder, requestContext, metadata, columnValue, resultKeyToAttributeMetadataMap, aggregates.isEmpty());
            }
        }
    }
    return new EntityFetcherResponse(entityBuilders);
}

Example 3 with AttributeMetadataProvider

use of org.hypertrace.gateway.service.common.AttributeMetadataProvider in project gateway-service by hypertrace.

the class EntityInteractionsFetcherTest method testServiceToServiceEdgeQueryRequests.

@Test
public void testServiceToServiceEdgeQueryRequests() {
    EntitiesRequest request = EntitiesRequest.newBuilder().setEntityType(DomainEntityType.SERVICE.name()).setStartTimeMillis(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30)).setEndTimeMillis(System.currentTimeMillis()).addSelection(buildAttributeExpression("SERVICE.name")).setIncomingInteractions(fromServiceInteractions).setOutgoingInteractions(toServiceInteractions).build();
    attributeMetadataProvider = mock(AttributeMetadataProvider.class);
    Mockito.when(attributeMetadataProvider.getAttributeMetadata(any(), Mockito.eq(AttributeScope.INTERACTION.name()), Mockito.eq("startTime"))).thenReturn(Optional.of(AttributeMetadata.newBuilder().setId("dummy").build()));
    EntityInteractionsFetcher aggregator = new EntityInteractionsFetcher(null, 500, attributeMetadataProvider);
    Map<String, QueryRequest> queryRequests = aggregator.buildQueryRequests(request.getStartTimeMillis(), request.getEndTimeMillis(), request.getSpaceId(), request.getEntityType(), request.getIncomingInteractions(), Collections.singleton(EntityKey.from("test_id1")), true, null);
    assertEquals(1, queryRequests.size());
    QueryRequest queryRequest = queryRequests.get("SERVICE");
    assertNotNull(queryRequest);
    assertEquals(2, queryRequest.getGroupByCount());
    assertEquals(fromServiceInteractions.getSelectionCount() + 2, queryRequest.getSelectionCount());
    System.out.println(queryRequest);
    queryRequests = aggregator.buildQueryRequests(request.getStartTimeMillis(), request.getEndTimeMillis(), request.getSpaceId(), request.getEntityType(), request.getOutgoingInteractions(), Collections.singleton(EntityKey.from("test_id1")), false, null);
    assertEquals(1, queryRequests.size());
    queryRequest = queryRequests.get("SERVICE");
    assertNotNull(queryRequest);
    assertEquals(2, queryRequest.getGroupByCount());
    assertEquals(toServiceInteractions.getSelectionCount() + 2, queryRequest.getSelectionCount());
    System.out.println(queryRequest);
}

Example 4 with AttributeMetadataProvider

use of org.hypertrace.gateway.service.common.AttributeMetadataProvider in project gateway-service by hypertrace.

the class EntityInteractionsFetcherTest method testFromMultipleEntitiesQuery.

@Test
public void testFromMultipleEntitiesQuery() {
    Set<String> entityTypes = ImmutableSet.of("SERVICE", "API");
    Set<String> entityIds = ImmutableSet.of("service_id_1", "api_id_1");
    Filter.Builder entityTypeFilter = Filter.newBuilder().setLhs(buildAttributeExpression("INTERACTION.fromEntityType")).setOperator(Operator.IN).setRhs(Expression.newBuilder().setLiteral(LiteralConstant.newBuilder().setValue(Value.newBuilder().setValueType(ValueType.STRING_ARRAY).addAllStringArray(entityTypes))));
    Filter.Builder entityIdFilter = Filter.newBuilder().setLhs(buildAttributeExpression("INTERACTION.fromEntityId")).setOperator(Operator.IN).setRhs(Expression.newBuilder().setLiteral(LiteralConstant.newBuilder().setValue(Value.newBuilder().setValueType(ValueType.STRING_ARRAY).addAllStringArray(entityIds))));
    InteractionsRequest fromInteraction = InteractionsRequest.newBuilder().setFilter(Filter.newBuilder().setOperator(Operator.AND).addChildFilter(entityTypeFilter).addChildFilter(entityIdFilter)).addSelection(getAggregateFunctionExpression("INTERACTION.bytesReceived", FunctionType.SUM, "SUM_bytes_received")).addSelection(getAggregateFunctionExpression("INTERACTION.bytesReceived", FunctionType.AVGRATE, "AVGRATE_bytes_received_60", ImmutableList.of(QueryExpressionUtil.getLiteralExpression(TimeUnit.MINUTES.toSeconds(1)).build()), false)).addSelection(getAggregateFunctionExpression("INTERACTION.bytesReceived", FunctionType.PERCENTILE, "p99_bytes_received", ImmutableList.of(QueryExpressionUtil.getLiteralExpression(Long.valueOf(99)).build()), false)).build();
    EntitiesRequest request = EntitiesRequest.newBuilder().setEntityType(DomainEntityType.SERVICE.name()).setStartTimeMillis(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30)).setEndTimeMillis(System.currentTimeMillis()).addSelection(buildAttributeExpression("SERVICE.name")).setIncomingInteractions(fromInteraction).build();
    attributeMetadataProvider = mock(AttributeMetadataProvider.class);
    Mockito.when(attributeMetadataProvider.getAttributeMetadata(any(), Mockito.eq(AttributeScope.INTERACTION.name()), Mockito.eq("startTime"))).thenReturn(Optional.of(AttributeMetadata.newBuilder().setId("dummy").build()));
    EntityInteractionsFetcher aggregator = new EntityInteractionsFetcher(null, 500, attributeMetadataProvider);
    Map<String, QueryRequest> queryRequests = aggregator.buildQueryRequests(request.getStartTimeMillis(), request.getEndTimeMillis(), request.getSpaceId(), request.getEntityType(), request.getIncomingInteractions(), Collections.singleton(EntityKey.from("test_id1")), true, null);
    assertEquals(entityTypes.size(), queryRequests.size());
    for (QueryRequest queryRequest : queryRequests.values()) {
        assertNotNull(queryRequest);
        assertEquals(2, queryRequest.getGroupByCount());
        assertEquals(fromInteraction.getSelectionCount() + 2, queryRequest.getSelectionCount());
        System.out.println(queryRequest);
    }
}

Example 5 with AttributeMetadataProvider

use of org.hypertrace.gateway.service.common.AttributeMetadataProvider in project gateway-service by hypertrace.

the class EntityInteractionsFetcherTest method testToMultipleEntityTypesQuery.

@Test
public void testToMultipleEntityTypesQuery() {
    Set<String> entityTypes = ImmutableSet.of("SERVICE", "BACKEND");
    Set<String> entityIds = ImmutableSet.of("service_id_1");
    Filter.Builder entityTypeFilter = Filter.newBuilder().setLhs(buildAttributeExpression("INTERACTION.toEntityType")).setOperator(Operator.IN).setRhs(Expression.newBuilder().setLiteral(LiteralConstant.newBuilder().setValue(Value.newBuilder().setValueType(ValueType.STRING_ARRAY).addAllStringArray(entityTypes))));
    InteractionsRequest toInteraction = InteractionsRequest.newBuilder().setFilter(entityTypeFilter).addSelection(getAggregateFunctionExpression("INTERACTION.bytesReceived", FunctionType.SUM, "SUM_bytes_received")).build();
    EntitiesRequest request = EntitiesRequest.newBuilder().setEntityType(DomainEntityType.SERVICE.name()).setStartTimeMillis(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30)).setEndTimeMillis(System.currentTimeMillis()).addSelection(buildAttributeExpression("SERVICE.name")).addSelection(buildAttributeExpression("INTERACTION.fromEntityType")).addSelection(buildAttributeExpression("INTERACTION.toEntityType")).setOutgoingInteractions(toInteraction).build();
    attributeMetadataProvider = mock(AttributeMetadataProvider.class);
    Mockito.when(attributeMetadataProvider.getAttributeMetadata(any(), Mockito.eq(AttributeScope.INTERACTION.name()), Mockito.eq("startTime"))).thenReturn(Optional.of(AttributeMetadata.newBuilder().setId("INTERACTION.startTime").build()));
    EntityInteractionsFetcher aggregator = new EntityInteractionsFetcher(null, 500, attributeMetadataProvider);
    Map<String, QueryRequest> queryRequests = aggregator.buildQueryRequests(request.getStartTimeMillis(), request.getEndTimeMillis(), request.getSpaceId(), request.getEntityType(), request.getOutgoingInteractions(), Collections.singleton(EntityKey.from("test_id1")), false, null);
    assertEquals(entityTypes.size(), queryRequests.size());
    for (QueryRequest queryRequest : queryRequests.values()) {
        assertNotNull(queryRequest);
        assertEquals(2, queryRequest.getGroupByCount());
        assertEquals(toInteraction.getSelectionCount() + 2, queryRequest.getSelectionCount());
        System.out.println(queryRequest);
    }
    // Fully assert the QueryService requests.
    // One request should query for Service -> Service edges.
    org.hypertrace.core.query.service.api.Filter filter = queryRequests.get("SERVICE").getFilter();
    org.hypertrace.core.query.service.api.Filter expectedFilter = org.hypertrace.core.query.service.api.Filter.newBuilder().setOperator(org.hypertrace.core.query.service.api.Operator.AND).addChildFilter(createBetweenTimesFilter("INTERACTION.startTime", request.getStartTimeMillis(), request.getEndTimeMillis())).addChildFilter(createStringArrayFilter(org.hypertrace.core.query.service.api.Operator.IN, "INTERACTION.fromServiceId", List.of("test_id1"))).addChildFilter(org.hypertrace.core.query.service.api.Filter.newBuilder().setOperator(org.hypertrace.core.query.service.api.Operator.AND).addChildFilter(createFilter("INTERACTION.toServiceId", org.hypertrace.core.query.service.api.Operator.NEQ, createStringNullLiteralExpression()))).build();
    assertEquals(expectedFilter, filter);
    // The other request should query for Service -> Backend edges.
    filter = queryRequests.get("BACKEND").getFilter();
    expectedFilter = org.hypertrace.core.query.service.api.Filter.newBuilder().setOperator(org.hypertrace.core.query.service.api.Operator.AND).addChildFilter(createBetweenTimesFilter("INTERACTION.startTime", request.getStartTimeMillis(), request.getEndTimeMillis())).addChildFilter(createStringArrayFilter(org.hypertrace.core.query.service.api.Operator.IN, "INTERACTION.fromServiceId", List.of("test_id1"))).addChildFilter(org.hypertrace.core.query.service.api.Filter.newBuilder().setOperator(org.hypertrace.core.query.service.api.Operator.AND).addChildFilter(createFilter("INTERACTION.toBackendId", org.hypertrace.core.query.service.api.Operator.NEQ, createStringNullLiteralExpression()))).build();
    assertEquals(expectedFilter, filter);
}