Search in sources :

Example 6 with Builder

use of org.hypertrace.entity.data.service.v1.Entity.Builder in project entity-service by hypertrace.

the class EntityDataServiceImpl method getAndUpsertEntities.

@Override
public void getAndUpsertEntities(Entities request, StreamObserver<Entity> responseObserver) {
    String tenantId = RequestContext.CURRENT.get().getTenantId().orElse(null);
    if (tenantId == null) {
        responseObserver.onError(new ServiceException("Tenant id is missing in the request."));
        return;
    }
    try {
        Map<Key, Document> documentMap = new HashMap<>();
        List<Entity> updatedEntities = new ArrayList<>();
        for (Entity entity : request.getEntityList()) {
            Entity normalizedEntity = this.entityNormalizer.normalize(tenantId, entity);
            updatedEntities.add(normalizedEntity);
            Document doc = convertEntityToDocument(normalizedEntity);
            Key key = this.entityNormalizer.getEntityDocKey(tenantId, normalizedEntity);
            documentMap.put(key, doc);
        }
        List<Entity> existingEntities = Streams.stream(entitiesCollection.bulkUpsertAndReturnOlderDocuments(documentMap)).flatMap(document -> PARSER.<Entity>parseOrLog(document, Entity.newBuilder()).stream()).map(Entity::toBuilder).map(builder -> builder.setTenantId(tenantId)).map(Entity.Builder::build).collect(Collectors.toList());
        existingEntities.forEach(responseObserver::onNext);
        responseObserver.onCompleted();
        entityChangeEventGenerator.sendChangeNotification(RequestContext.CURRENT.get(), existingEntities, updatedEntities);
    } catch (IOException e) {
        LOG.error("Failed to bulk upsert entities", e);
        responseObserver.onError(e);
    }
}
Also used : EnrichedEntity(org.hypertrace.entity.data.service.v1.EnrichedEntity) Entity(org.hypertrace.entity.data.service.v1.Entity) ServiceException(com.google.protobuf.ServiceException) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) IOException(java.io.IOException) Document(org.hypertrace.core.documentstore.Document) JSONDocument(org.hypertrace.core.documentstore.JSONDocument) Key(org.hypertrace.core.documentstore.Key)

Example 7 with Builder

use of org.hypertrace.entity.data.service.v1.Entity.Builder in project entity-service by hypertrace.

the class EntityDataServiceImpl method mergeAndUpsertEntity.

@Override
public void mergeAndUpsertEntity(MergeAndUpsertEntityRequest request, StreamObserver<MergeAndUpsertEntityResponse> responseObserver) {
    RequestContext requestContext = RequestContext.CURRENT.get();
    String tenantId = requestContext.getTenantId().orElse(null);
    if (tenantId == null) {
        responseObserver.onError(new ServiceException("Tenant id is missing in the request."));
        return;
    }
    Entity receivedEntity = this.entityNormalizer.normalize(tenantId, request.getEntity());
    Optional<Entity> existingEntity = getExistingEntity(tenantId, receivedEntity.getEntityType(), receivedEntity.getEntityId());
    boolean rejectUpsertForConditionMismatch = existingEntity.map(entity -> !this.upsertConditionMatcher.matches(entity, request.getUpsertCondition())).orElse(false);
    if (rejectUpsertForConditionMismatch) {
        // There's an existing entity and the update doesn't meet the condition, return existing
        responseObserver.onNext(MergeAndUpsertEntityResponse.newBuilder().setEntity(existingEntity.get()).build());
        responseObserver.onCompleted();
    } else {
        // There's either a new entity or a valid update to upsert
        Entity entityToUpsert = existingEntity.map(Entity::toBuilder).map(Entity.Builder::clearCreatedTime).map(builder -> builder.mergeFrom(receivedEntity)).map(Builder::build).orElse(receivedEntity);
        try {
            Entity upsertedEntity = this.upsertEntity(tenantId, entityToUpsert);
            responseObserver.onNext(MergeAndUpsertEntityResponse.newBuilder().setEntity(upsertedEntity).build());
            responseObserver.onCompleted();
            entityChangeEventGenerator.sendChangeNotification(requestContext, existingEntity.map(List::of).orElse(Collections.emptyList()), List.of(upsertedEntity));
        } catch (IOException e) {
            responseObserver.onError(e);
        }
    }
}
Also used : ByIdRequest(org.hypertrace.entity.data.service.v1.ByIdRequest) ServiceException(com.google.protobuf.ServiceException) LoggerFactory(org.slf4j.LoggerFactory) Channel(io.grpc.Channel) EntityTypeClient(org.hypertrace.entity.type.service.rxclient.EntityTypeClient) StreamObserver(io.grpc.stub.StreamObserver) Map(java.util.Map) Objects.isNull(java.util.Objects.isNull) ENRICHED_ENTITIES_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.ENRICHED_ENTITIES_COLLECTION) DocStoreConverter(org.hypertrace.entity.service.util.DocStoreConverter) InvalidRequestException(org.hypertrace.entity.service.exception.InvalidRequestException) StringUtils(org.hypertrace.entity.service.util.StringUtils) Streams(com.google.common.collect.Streams) Empty(org.hypertrace.entity.data.service.v1.Empty) Collectors(java.util.stream.Collectors) List(java.util.List) Stream(java.util.stream.Stream) EntityRelationship(org.hypertrace.entity.data.service.v1.EntityRelationship) EntityRelationships(org.hypertrace.entity.data.service.v1.EntityRelationships) MergeAndUpsertEntityRequest(org.hypertrace.entity.data.service.v1.MergeAndUpsertEntityRequest) Optional(java.util.Optional) EntityChangeEventGenerator(org.hypertrace.entity.service.change.event.api.EntityChangeEventGenerator) EnrichedEntity(org.hypertrace.entity.data.service.v1.EnrichedEntity) Builder(org.hypertrace.entity.data.service.v1.Entity.Builder) Document(org.hypertrace.core.documentstore.Document) RequestContext(org.hypertrace.core.grpcutils.context.RequestContext) ByTypeAndIdentifyingAttributes(org.hypertrace.entity.data.service.v1.ByTypeAndIdentifyingAttributes) DocStoreJsonFormat(org.hypertrace.entity.service.util.DocStoreJsonFormat) Descriptors(com.google.protobuf.Descriptors) Filter(org.hypertrace.core.documentstore.Filter) HashMap(java.util.HashMap) Function(java.util.function.Function) Collection(org.hypertrace.core.documentstore.Collection) Entities(org.hypertrace.entity.data.service.v1.Entities) ArrayList(java.util.ArrayList) Datastore(org.hypertrace.core.documentstore.Datastore) Key(org.hypertrace.core.documentstore.Key) EntityServiceConstants(org.hypertrace.entity.service.constants.EntityServiceConstants) Entity(org.hypertrace.entity.data.service.v1.Entity) EntityDataServiceImplBase(org.hypertrace.entity.data.service.v1.EntityDataServiceGrpc.EntityDataServiceImplBase) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) JSONDocument(org.hypertrace.core.documentstore.JSONDocument) IOException(java.io.IOException) ENTITY_RELATIONSHIPS_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.ENTITY_RELATIONSHIPS_COLLECTION) EnrichedEntities(org.hypertrace.entity.data.service.v1.EnrichedEntities) Query(org.hypertrace.entity.data.service.v1.Query) RAW_ENTITIES_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.RAW_ENTITIES_COLLECTION) Message(com.google.protobuf.Message) GeneratedMessageV3(com.google.protobuf.GeneratedMessageV3) MergeAndUpsertEntityResponse(org.hypertrace.entity.data.service.v1.MergeAndUpsertEntityResponse) Collections(java.util.Collections) RelationshipsQuery(org.hypertrace.entity.data.service.v1.RelationshipsQuery) EnrichedEntity(org.hypertrace.entity.data.service.v1.EnrichedEntity) Entity(org.hypertrace.entity.data.service.v1.Entity) ServiceException(com.google.protobuf.ServiceException) List(java.util.List) ArrayList(java.util.ArrayList) RequestContext(org.hypertrace.core.grpcutils.context.RequestContext) IOException(java.io.IOException)

Example 8 with Builder

use of org.hypertrace.entity.data.service.v1.Entity.Builder in project entity-service by hypertrace.

the class EntityDataServiceImpl method searchByIdAndStreamSingleResponse.

private <T extends Message> void searchByIdAndStreamSingleResponse(String tenantId, String entityId, String entityType, Collection collection, Message.Builder builder, StreamObserver<T> responseObserver) {
    org.hypertrace.core.documentstore.Query query = new org.hypertrace.core.documentstore.Query();
    String docId = this.entityNormalizer.getEntityDocKey(tenantId, entityType, entityId).toString();
    query.setFilter(new Filter(Filter.Op.EQ, EntityServiceConstants.ID, docId));
    Iterator<Document> result = collection.search(query);
    List<T> entities = new ArrayList<>();
    while (result.hasNext()) {
        PARSER.<T>parseOrLog(result.next(), builder.clone()).map(entity -> {
            // Populate the tenant id field with the tenant id that's received for backward
            // compatibility.
            Descriptors.FieldDescriptor fieldDescriptor = entity.getDescriptorForType().findFieldByName("tenant_id");
            if (fieldDescriptor != null) {
                return (T) entity.toBuilder().setField(fieldDescriptor, tenantId).build();
            }
            return entity;
        }).ifPresent(entities::add);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Docstore query has returned the result: {}", entities);
    }
    if (entities.size() == 1) {
        responseObserver.onNext(entities.get(0));
        responseObserver.onCompleted();
    } else if (entities.size() > 1) {
        responseObserver.onError(new IllegalStateException("Multiple entities with same id are found."));
    } else {
        // When there is no result, we should return the default instance, which is a way
        // of saying it's null.
        // TODO : Not convinced with the default instance
        responseObserver.onNext((T) builder.build());
        responseObserver.onCompleted();
    }
}
Also used : ByIdRequest(org.hypertrace.entity.data.service.v1.ByIdRequest) ServiceException(com.google.protobuf.ServiceException) LoggerFactory(org.slf4j.LoggerFactory) Channel(io.grpc.Channel) EntityTypeClient(org.hypertrace.entity.type.service.rxclient.EntityTypeClient) StreamObserver(io.grpc.stub.StreamObserver) Map(java.util.Map) Objects.isNull(java.util.Objects.isNull) ENRICHED_ENTITIES_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.ENRICHED_ENTITIES_COLLECTION) DocStoreConverter(org.hypertrace.entity.service.util.DocStoreConverter) InvalidRequestException(org.hypertrace.entity.service.exception.InvalidRequestException) StringUtils(org.hypertrace.entity.service.util.StringUtils) Streams(com.google.common.collect.Streams) Empty(org.hypertrace.entity.data.service.v1.Empty) Collectors(java.util.stream.Collectors) List(java.util.List) Stream(java.util.stream.Stream) EntityRelationship(org.hypertrace.entity.data.service.v1.EntityRelationship) EntityRelationships(org.hypertrace.entity.data.service.v1.EntityRelationships) MergeAndUpsertEntityRequest(org.hypertrace.entity.data.service.v1.MergeAndUpsertEntityRequest) Optional(java.util.Optional) EntityChangeEventGenerator(org.hypertrace.entity.service.change.event.api.EntityChangeEventGenerator) EnrichedEntity(org.hypertrace.entity.data.service.v1.EnrichedEntity) Builder(org.hypertrace.entity.data.service.v1.Entity.Builder) Document(org.hypertrace.core.documentstore.Document) RequestContext(org.hypertrace.core.grpcutils.context.RequestContext) ByTypeAndIdentifyingAttributes(org.hypertrace.entity.data.service.v1.ByTypeAndIdentifyingAttributes) DocStoreJsonFormat(org.hypertrace.entity.service.util.DocStoreJsonFormat) Descriptors(com.google.protobuf.Descriptors) Filter(org.hypertrace.core.documentstore.Filter) HashMap(java.util.HashMap) Function(java.util.function.Function) Collection(org.hypertrace.core.documentstore.Collection) Entities(org.hypertrace.entity.data.service.v1.Entities) ArrayList(java.util.ArrayList) Datastore(org.hypertrace.core.documentstore.Datastore) Key(org.hypertrace.core.documentstore.Key) EntityServiceConstants(org.hypertrace.entity.service.constants.EntityServiceConstants) Entity(org.hypertrace.entity.data.service.v1.Entity) EntityDataServiceImplBase(org.hypertrace.entity.data.service.v1.EntityDataServiceGrpc.EntityDataServiceImplBase) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) JSONDocument(org.hypertrace.core.documentstore.JSONDocument) IOException(java.io.IOException) ENTITY_RELATIONSHIPS_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.ENTITY_RELATIONSHIPS_COLLECTION) EnrichedEntities(org.hypertrace.entity.data.service.v1.EnrichedEntities) Query(org.hypertrace.entity.data.service.v1.Query) RAW_ENTITIES_COLLECTION(org.hypertrace.entity.service.constants.EntityCollectionConstants.RAW_ENTITIES_COLLECTION) Message(com.google.protobuf.Message) GeneratedMessageV3(com.google.protobuf.GeneratedMessageV3) MergeAndUpsertEntityResponse(org.hypertrace.entity.data.service.v1.MergeAndUpsertEntityResponse) Collections(java.util.Collections) RelationshipsQuery(org.hypertrace.entity.data.service.v1.RelationshipsQuery) Query(org.hypertrace.entity.data.service.v1.Query) RelationshipsQuery(org.hypertrace.entity.data.service.v1.RelationshipsQuery) ArrayList(java.util.ArrayList) Document(org.hypertrace.core.documentstore.Document) JSONDocument(org.hypertrace.core.documentstore.JSONDocument) Filter(org.hypertrace.core.documentstore.Filter)

Example 9 with Builder

use of org.hypertrace.entity.data.service.v1.Entity.Builder in project entity-service by hypertrace.

the class EntityDataServiceTest method testUpdateEntityViaUpsertAndMerge.

@Test
public void testUpdateEntityViaUpsertAndMerge() {
    // Scope this test to its own tenant for isolation
    String TENANT_ID = EntityDataServiceTest.TENANT_ID + "_testUpdateEntityViaUpsertAndMerge";
    EntityDataServiceBlockingStub entityDataServiceStub = buildStubForTenant(TENANT_ID);
    setupEntityTypes(channel, TENANT_ID);
    Map<String, AttributeValue> createAttributes = Map.of("some-attr", stringValue("v1"));
    Map<String, AttributeValue> updateAttributes = Map.of("some-other-attr", stringValue("v2"));
    // V1 entity
    Entity entityToCreate = Entity.newBuilder().setTenantId(TENANT_ID).setEntityType(EntityType.K8S_POD.name()).setEntityName("V1 UPDATE POD").putIdentifyingAttributes(EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_EXTERNAL_ID), generateRandomUUIDAttrValue()).putAllAttributes(createAttributes).build();
    entityDataServiceStub.mergeAndUpsertEntity(MergeAndUpsertEntityRequest.newBuilder().setEntity(entityToCreate).build());
    Entity entityUpdate = entityToCreate.toBuilder().clearAttributes().putAllAttributes(updateAttributes).build();
    Entity updatedEntity = entityDataServiceStub.mergeAndUpsertEntity(MergeAndUpsertEntityRequest.newBuilder().setEntity(entityUpdate).build()).getEntity();
    Map<String, AttributeValue> combinedAttributes = ImmutableMap.<String, AttributeValue>builder().putAll(createAttributes).putAll(entityToCreate.getIdentifyingAttributesMap()).putAll(updateAttributes).build();
    assertEquals(combinedAttributes, updatedEntity.getAttributesMap());
    // V2 Entity
    entityToCreate = Entity.newBuilder().setTenantId(TENANT_ID).setEntityId(UUID.randomUUID().toString()).setEntityType(TEST_ENTITY_TYPE_V2).setEntityName("V2 entity update").putAllAttributes(createAttributes).build();
    entityDataServiceStub.mergeAndUpsertEntity(MergeAndUpsertEntityRequest.newBuilder().setEntity(entityToCreate).build());
    entityUpdate = entityToCreate.toBuilder().clearAttributes().putAllAttributes(updateAttributes).build();
    updatedEntity = entityDataServiceStub.mergeAndUpsertEntity(MergeAndUpsertEntityRequest.newBuilder().setEntity(entityUpdate).build()).getEntity();
    combinedAttributes = ImmutableMap.<String, AttributeValue>builder().putAll(createAttributes).putAll(updateAttributes).build();
    assertEntityEquals(entityToCreate.toBuilder().clearAttributes().putAllAttributes(combinedAttributes).build(), updatedEntity);
}
Also used : EnrichedEntity(org.hypertrace.entity.data.service.v1.EnrichedEntity) Entity(org.hypertrace.entity.data.service.v1.Entity) AttributeValue(org.hypertrace.entity.data.service.v1.AttributeValue) EntityDataServiceBlockingStub(org.hypertrace.entity.data.service.v1.EntityDataServiceGrpc.EntityDataServiceBlockingStub) Test(org.junit.jupiter.api.Test)

Aggregations

Entity (org.hypertrace.entity.data.service.v1.Entity)5 Builder (org.hypertrace.entity.data.service.v1.Entity.Builder)5 HashMap (java.util.HashMap)4 EnrichedEntity (org.hypertrace.entity.data.service.v1.EnrichedEntity)4 ServiceException (com.google.protobuf.ServiceException)3 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 Document (org.hypertrace.core.documentstore.Document)3 JSONDocument (org.hypertrace.core.documentstore.JSONDocument)3 Key (org.hypertrace.core.documentstore.Key)3 Streams (com.google.common.collect.Streams)2 Descriptors (com.google.protobuf.Descriptors)2 GeneratedMessageV3 (com.google.protobuf.GeneratedMessageV3)2 Message (com.google.protobuf.Message)2 Channel (io.grpc.Channel)2 StreamObserver (io.grpc.stub.StreamObserver)2 Collections (java.util.Collections)2 Iterator (java.util.Iterator)2 List (java.util.List)2 Map (java.util.Map)2