Examples with EntityProto com.google.storage.onestore.v3.OnestoreEntity.EntityProto used on opensource projects

Example 31 with EntityProto

use of com.google.storage.onestore.v3.OnestoreEntity.EntityProto in project appengine-java-standard by GoogleCloudPlatform.

the class LocalDatastoreService method commitImpl.

/**
 * Requires a lock on the provided profile.
 */
private CommitResponse commitImpl(LiveTxn liveTxn, final Profile profile) {
    // assumes we already have a lock on the profile
    CommitResponse response = new CommitResponse();
    for (EntityGroupTracker tracker : liveTxn.getAllTrackers()) {
        // This will throw an exception if the entity group
        // has been modified since this transaction started.
        tracker.checkEntityGroupVersion();
    }
    int deleted = 0;
    int written = 0;
    Cost totalCost = new Cost();
    long commitTimestamp = profile.incrementAndGetCommitTimestamp();
    for (EntityGroupTracker tracker : liveTxn.getAllTrackers()) {
        Profile.EntityGroup eg = tracker.getEntityGroup();
        eg.incrementVersion();
        final Collection<EntityProto> writtenEntities = tracker.getWrittenEntities();
        final Collection<Reference> deletedKeys = tracker.getDeletedKeys();
        LocalDatastoreJob job = new WriteJob(highRepJobPolicy, eg, profile, commitTimestamp, writtenEntities, deletedKeys);
        addTo(totalCost, job.calculateJobCost());
        eg.addJob(job);
        deleted += deletedKeys.size();
        written += writtenEntities.size();
        for (EntityProto writtenEntity : writtenEntities) {
            response.addVersion().setRootEntityKey(writtenEntity.getKey()).setVersion(job.getMutationTimestamp(writtenEntity.getKey()));
        }
        for (Reference deletedKey : deletedKeys) {
            response.addVersion().setRootEntityKey(deletedKey).setVersion(job.getMutationTimestamp(deletedKey));
        }
    }
    logger.fine("committed: " + written + " puts, " + deleted + " deletes in " + liveTxn.getAllTrackers().size() + " entity groups");
    response.setCost(totalCost);
    return response;
}

Example 32 with EntityProto

use of com.google.storage.onestore.v3.OnestoreEntity.EntityProto in project appengine-java-standard by GoogleCloudPlatform.

the class LocalDatastoreService method deleteImpl.

// status
@SuppressWarnings("unused")
public DeleteResponse deleteImpl(Status status, DeleteRequest request) {
    DeleteResponse response = new DeleteResponse();
    if (request.keySize() == 0) {
        return response;
    }
    Cost totalCost = response.getMutableCost();
    // We don't support requests that span apps, so the app for the first key
    // is the app for all keys.
    String app = request.keys().get(0).getApp();
    final Profile profile = getOrCreateProfile(app);
    LiveTxn liveTxn = null;
    // Maintain a mapping of keys by entity group so that we can apply one job
    // per entity group.
    Map<Path, List<Reference>> keysByEntityGroup = new LinkedHashMap<>();
    Map<Reference, Long> writtenVersions = new HashMap<>();
    synchronized (profile) {
        for (final Reference key : request.keys()) {
            validatePathComplete(key);
            Path group = getGroup(key);
            if (request.hasTransaction()) {
                if (liveTxn == null) {
                    liveTxn = profile.getTxn(request.getTransaction().getHandle());
                }
                checkRequest(!liveTxn.isReadOnly(), "Cannot modify entities in a read-only transaction.");
                Profile.EntityGroup eg = profile.getGroup(group);
                // this will throw an exception if we attempt to modify
                // the wrong entity group
                eg.addTransaction(liveTxn).addDeletedEntity(key);
            } else {
                List<Reference> keysToDelete = keysByEntityGroup.get(group);
                if (keysToDelete == null) {
                    keysToDelete = new ArrayList<>();
                    keysByEntityGroup.put(group, keysToDelete);
                }
                keysToDelete.add(key);
            }
        }
        // does all the work for each entity group.
        for (final Map.Entry<Path, List<Reference>> entry : keysByEntityGroup.entrySet()) {
            Profile.EntityGroup eg = profile.getGroup(entry.getKey());
            eg.incrementVersion();
            LocalDatastoreJob job = new WriteJob(highRepJobPolicy, eg, profile, Collections.<EntityProto>emptyList(), entry.getValue());
            addTo(totalCost, job.calculateJobCost());
            eg.addJob(job);
            for (Reference deletedKey : entry.getValue()) {
                writtenVersions.put(deletedKey, job.getMutationTimestamp(deletedKey));
            }
        }
    }
    if (!request.hasTransaction()) {
        for (Reference key : request.keys()) {
            response.addVersion(writtenVersions.get(key));
        }
    }
    return response;
}

Example 33 with EntityProto

use of com.google.storage.onestore.v3.OnestoreEntity.EntityProto in project appengine-java-standard by GoogleCloudPlatform.

the class LocalDatastoreService method get.

public GetResponse get(@SuppressWarnings("unused") Status status, GetRequest request) {
    GetResponse response = new GetResponse();
    LiveTxn liveTxn = null;
    for (Reference key : request.keys()) {
        validatePathComplete(key);
        String app = key.getApp();
        Path groupPath = getGroup(key);
        GetResponse.Entity responseEntity = response.addEntity();
        Profile profile = getOrCreateProfile(app);
        synchronized (profile) {
            Profile.EntityGroup eg = profile.getGroup(groupPath);
            if (request.hasTransaction()) {
                if (liveTxn == null) {
                    liveTxn = profile.getTxn(request.getTransaction().getHandle());
                }
                // this will throw an exception if we attempt to read from
                // the wrong entity group
                eg.addTransaction(liveTxn);
            }
            boolean eventualConsistency = request.hasFailoverMs() && liveTxn == null;
            EntityProto entity = pseudoKinds.get(liveTxn, eg, key, eventualConsistency);
            if (entity == PseudoKinds.NOT_A_PSEUDO_KIND) {
                VersionedEntity versionedEntity = eg.get(liveTxn, key, eventualConsistency);
                if (versionedEntity == null) {
                    entity = null;
                    if (!eventualConsistency) {
                        responseEntity.setVersion(profile.getReadTimestamp());
                    }
                } else {
                    entity = versionedEntity.entityProto();
                    responseEntity.setVersion(versionedEntity.version());
                }
            }
            if (entity != null) {
                responseEntity.getMutableEntity().copyFrom(entity);
                postprocessEntity(responseEntity.getMutableEntity());
            } else {
                responseEntity.getMutableKey().copyFrom(key);
            }
            // Give all entity groups with unapplied jobs the opportunity to catch
            // up.  Note that this will not impact the result we're about to return.
            profile.groom();
        }
    }
    return response;
}

Example 34 with EntityProto

use of com.google.storage.onestore.v3.OnestoreEntity.EntityProto in project appengine-java-standard by GoogleCloudPlatform.

the class LocalDatastoreService method createIndexEntities.

/**
 * Splits a full entity into all index entities seen in a projection.
 *
 * @param entity the entity to split
 * @param postfixProps the properties included in the postfix
 * @return A list of the index entities.
 */
private ImmutableList<EntityProto> createIndexEntities(EntityProto entity, Set<String> postfixProps, EntityProtoComparator entityComparator) {
    SetMultimap<String, PropertyValue> toSplit = MultimapBuilder.hashKeys(postfixProps.size()).hashSetValues(1).build();
    Set<String> seen = Sets.newHashSet();
    boolean splitRequired = false;
    for (Property prop : entity.propertys()) {
        if (postfixProps.contains(prop.getName())) {
            // If we have multiple values for any postfix property, we need to split.
            splitRequired |= !seen.add(prop.getName());
            // Only add the value if it matches the query filters
            if (entityComparator.matches(prop)) {
                toSplit.put(prop.getName(), prop.getValue());
            }
        }
    }
    if (!splitRequired) {
        // No need for splitting!
        return ImmutableList.of(entity);
    }
    EntityProto clone = new EntityProto();
    clone.getMutableKey().copyFrom(entity.getKey());
    clone.getMutableEntityGroup();
    List<EntityProto> results = Lists.newArrayList(clone);
    for (Map.Entry<String, Collection<PropertyValue>> entry : toSplit.asMap().entrySet()) {
        if (entry.getValue().size() == 1) {
            // No need for cloning!
            for (EntityProto result : results) {
                result.addProperty().setName(entry.getKey()).setMeaning(Property.Meaning.INDEX_VALUE).getMutableValue().copyFrom(Iterables.getOnlyElement(entry.getValue()));
            }
            continue;
        }
        List<EntityProto> splitResults = Lists.newArrayListWithCapacity(results.size() * entry.getValue().size());
        for (PropertyValue value : entry.getValue()) {
            for (EntityProto result : results) {
                EntityProto split = result.clone();
                split.addProperty().setName(entry.getKey()).setMeaning(Property.Meaning.INDEX_VALUE).getMutableValue().copyFrom(value);
                splitResults.add(split);
            }
        }
        results = splitResults;
    }
    return ImmutableList.copyOf(results);
}

Example 35 with EntityProto

use of com.google.storage.onestore.v3.OnestoreEntity.EntityProto in project appengine-java-standard by GoogleCloudPlatform.

the class KindPseudoKind method makeKindEntity.

/**
 * Creates a __kind__ entity
 */
private static EntityProto makeKindEntity(String kind, String app, String namespace) {
    EntityProto kindEntity = new EntityProto();
    Path path = new Path();
    path.addElement().setType(KIND_METADATA_KIND).setName(kind);
    Reference key = new Reference().setApp(app).setPath(path);
    if (namespace.length() > 0) {
        key.setNameSpace(namespace);
    }
    kindEntity.setKey(key);
    // EntityProto.entity_group is a required PB field.
    kindEntity.getMutableEntityGroup().addElement(path.getElement(0));
    return kindEntity;
}