use of com.google.storage.onestore.v3.OnestoreEntity.Reference in project appengine-java-standard by GoogleCloudPlatform.
the class ValidatedQuery method validateQuery.
/**
* Determines if a given query is supported by the datastore.
*
* @throws IllegalQueryException If the provided query fails validation.
*/
private void validateQuery() {
if (query.propertyNameSize() > 0 && query.isKeysOnly()) {
throw new IllegalQueryException("projection and keys_only cannot both be set", IllegalQueryType.ILLEGAL_PROJECTION);
}
Set<String> projectionProperties = new HashSet<String>(query.propertyNameSize());
for (String property : query.propertyNames()) {
if (!projectionProperties.add(property)) {
throw new IllegalQueryException("cannot project a property multiple times", IllegalQueryType.ILLEGAL_PROJECTION);
}
}
Set<String> groupBySet = Sets.newHashSetWithExpectedSize(query.groupByPropertyNameSize());
for (String name : query.groupByPropertyNames()) {
if (!groupBySet.add(name)) {
throw new IllegalQueryException("cannot group by a property multiple times", IllegalQueryType.ILLEGAL_GROUPBY);
}
// filters.
if (Entity.RESERVED_NAME.matcher(name).matches()) {
throw new IllegalQueryException("group by is not supported for the property: " + name, IllegalQueryType.ILLEGAL_GROUPBY);
}
}
/* Validate group by properties in orderings. */
Set<String> groupByInOrderSet = Sets.newHashSetWithExpectedSize(query.groupByPropertyNameSize());
for (Order order : query.orders()) {
if (groupBySet.contains(order.getProperty())) {
groupByInOrderSet.add(order.getProperty());
} else if (groupByInOrderSet.size() != groupBySet.size()) {
throw new IllegalQueryException("must specify all group by orderings before any non group by orderings", IllegalQueryType.ILLEGAL_GROUPBY);
}
}
// Transaction requires ancestor
if (query.hasTransaction() && !query.hasAncestor()) {
throw new IllegalQueryException("Only ancestor queries are allowed inside transactions.", IllegalQueryType.TRANSACTION_REQUIRES_ANCESTOR);
}
// Filters and sort orders require kind.
if (!query.hasKind()) {
for (Filter filter : query.filters()) {
if (!filter.getProperty(0).getName().equals(Entity.KEY_RESERVED_PROPERTY)) {
throw new IllegalQueryException("kind is required for non-__key__ filters", IllegalQueryType.KIND_REQUIRED);
}
}
for (Order order : query.orders()) {
if (!(order.getProperty().equals(Entity.KEY_RESERVED_PROPERTY) && order.getDirection() == Order.Direction.ASCENDING.getValue())) {
throw new IllegalQueryException("kind is required for all orders except __key__ ascending", IllegalQueryType.KIND_REQUIRED);
}
}
}
/* Validate ancestor, if it exists. */
if (query.hasAncestor()) {
Reference ancestor = query.getAncestor();
if (!ancestor.getApp().equals(query.getApp())) {
throw new IllegalQueryException("The query app is " + query.getApp() + " but ancestor app is " + ancestor.getApp(), IllegalQueryType.ILLEGAL_VALUE);
}
if (!ancestor.getNameSpace().equals(query.getNameSpace())) {
throw new IllegalQueryException("The query namespace is " + query.getNameSpace() + " but ancestor namespace is " + ancestor.getNameSpace(), IllegalQueryType.ILLEGAL_VALUE);
}
}
// Check for unsupported filter values. We only support one property
// per filter and one property with an inequality filter.
String ineqProp = null;
this.isGeo = false;
for (Filter filter : query.filters()) {
int numProps = filter.propertySize();
if (numProps != 1) {
throw new IllegalQueryException(String.format("Filter has %s properties, expected 1", numProps), IllegalQueryType.FILTER_WITH_MULTIPLE_PROPS);
}
Property prop = filter.getProperty(0);
String propName = prop.getName();
/* Extra validation for __key__. The filter value must be a key,
* if the query has a kind, the key's kind must match, and the
* app and namespace must match the query. */
if (Entity.KEY_RESERVED_PROPERTY.equals(propName)) {
PropertyValue value = prop.getValue();
if (!value.hasReferenceValue()) {
throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter value must be a Key", IllegalQueryType.ILLEGAL_VALUE);
}
ReferenceValue refVal = value.getReferenceValue();
if (!refVal.getApp().equals(query.getApp())) {
throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter app is " + refVal.getApp() + " but query app is " + query.getApp(), IllegalQueryType.ILLEGAL_VALUE);
}
if (!refVal.getNameSpace().equals(query.getNameSpace())) {
throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter namespace is " + refVal.getNameSpace() + " but query namespace is " + query.getNameSpace(), IllegalQueryType.ILLEGAL_VALUE);
}
}
if (INEQUALITY_OPERATORS.contains(filter.getOpEnum())) {
if (ineqProp == null) {
ineqProp = propName;
} else if (!ineqProp.equals(propName)) {
throw new IllegalQueryException(String.format("Only one inequality filter per query is supported. " + "Encountered both %s and %s", ineqProp, propName), IllegalQueryType.MULTIPLE_INEQ_FILTERS);
}
} else if (filter.getOpEnum() == Operator.EQUAL) {
if (projectionProperties.contains(propName)) {
throw new IllegalQueryException("cannot use projection on a property with an equality filter", IllegalQueryType.ILLEGAL_PROJECTION);
} else if (groupBySet.contains(propName)) {
throw new IllegalQueryException("cannot use group by on a property with an equality filter", IllegalQueryType.ILLEGAL_GROUPBY);
}
} else if (filter.getOpEnum() == Operator.CONTAINED_IN_REGION) {
isGeo = true;
if (!filter.hasGeoRegion() || prop.getValue().hasPointValue()) {
throw new IllegalQueryException(String.format("Geo-spatial filter on %s should specify GeoRegion rather than Property Value", propName), IllegalQueryType.UNSUPPORTED_FILTER);
}
GeoRegion region = filter.getGeoRegion();
if ((region.hasCircle() && region.hasRectangle()) || (!region.hasCircle() && !region.hasRectangle())) {
throw new IllegalQueryException(String.format("Geo-spatial filter on %s should specify Circle or Rectangle, but not both", propName), IllegalQueryType.UNSUPPORTED_FILTER);
}
} else if (UNSUPPORTED_OPERATORS.contains(filter.getOpEnum())) {
throw new IllegalQueryException(String.format("Unsupported filter operator: %s", filter.getOp()), IllegalQueryType.UNSUPPORTED_FILTER);
}
}
if (isGeo) {
if (ineqProp != null) {
throw new IllegalQueryException("Inequality filter with geo-spatial query is not supported.", IllegalQueryType.UNSUPPORTED_FILTER);
}
if (query.hasAncestor()) {
throw new IllegalQueryException("Geo-spatial filter on ancestor query is not supported.", IllegalQueryType.UNSUPPORTED_FILTER);
}
if (query.hasCompiledCursor() || query.hasEndCompiledCursor()) {
throw new IllegalQueryException("Start and end cursors are not supported on geo-spatial queries.", IllegalQueryType.CURSOR_NOT_SUPPORTED);
}
}
if (ineqProp != null && query.groupByPropertyNameSize() > 0) {
if (!groupBySet.contains(ineqProp)) {
throw new IllegalQueryException(String.format("Inequality filter on %s must also be a group by property when " + "group by properties are set.", ineqProp), IllegalQueryType.ILLEGAL_GROUPBY);
}
}
if (ineqProp != null) {
if (query.orderSize() > 0) {
if (!ineqProp.equals(query.getOrder(0).getProperty())) {
// First order must match the inequality filter.
throw new IllegalQueryException(String.format("The first sort property must be the same as the property to which " + "the inequality filter is applied. In your query the first sort property " + "is %s but the inequality filter is on %s", query.getOrder(0).getProperty(), ineqProp), IllegalQueryType.FIRST_SORT_NEQ_INEQ_PROP);
}
}
}
}
use of com.google.storage.onestore.v3.OnestoreEntity.Reference in project appengine-java-standard by GoogleCloudPlatform.
the class LocalDatastoreService method getGroup.
/**
* Returns the entity group for the specified key. This is simply a new {@code Path} instance
* containing the first element in the specified key.
*/
private Path getGroup(Reference key) {
Path path = key.getPath();
Path group = new Path();
group.addElement(path.getElement(0));
return group;
}
use of com.google.storage.onestore.v3.OnestoreEntity.Reference in project appengine-java-standard by GoogleCloudPlatform.
the class LocalDatastoreService method runQuery.
// status
@SuppressWarnings("unused")
public QueryResult runQuery(Status status, Query query) {
// Construct a validated query right away so we can fail fast
// if something is wrong.
final LocalCompositeIndexManager.ValidatedQuery validatedQuery = new LocalCompositeIndexManager.ValidatedQuery(query);
query = validatedQuery.getV3Query();
// LocalCompositeIndexManager.ValidatedQuery. I don't know why.
try {
CursorModernizer.modernizeQueryCursors(query);
} catch (InvalidConversionException e) {
throw newError(ErrorCode.BAD_REQUEST, "Invalid cursor");
}
String app = query.getApp();
Profile profile = getOrCreateProfile(app);
synchronized (profile) {
// ancestor does not imply we have a transaction.
if (query.hasTransaction() || query.hasAncestor()) {
// Query can only have a txn if it is an ancestor query. Either way we
// know we've got an ancestor.
Path groupPath = getGroup(query.getAncestor());
Profile.EntityGroup eg = profile.getGroup(groupPath);
if (query.hasTransaction()) {
if (!app.equals(query.getTransaction().getApp())) {
throw newError(ErrorCode.INTERNAL_ERROR, "Can't query app " + app + "in a transaction on app " + query.getTransaction().getApp());
}
LiveTxn liveTxn = profile.getTxn(query.getTransaction().getHandle());
// this will throw an exception if we attempt to read from
// the wrong entity group
eg.addTransaction(liveTxn);
// Use snapshot profile.
profile = eg.getSnapshot(liveTxn);
}
if (query.hasAncestor()) {
if (query.hasTransaction() || !query.hasFailoverMs()) {
// Either we have a transaction or the user has requested strongly
// consistent results. Either way, we need to apply jobs.
eg.rollForwardUnappliedJobs();
}
}
}
if (query.hasSearchQuery()) {
throw newError(ErrorCode.BAD_REQUEST, "full-text search unsupported");
}
// Run as a PseudoKind query if necessary, otherwise check the actual local datastore
List<EntityProto> queryEntities = pseudoKinds.runQuery(query);
Map<Reference, Long> versions = null;
if (queryEntities == null) {
Collection<VersionedEntity> versionedEntities = null;
Map<String, Extent> extents = profile.getExtents();
Extent extent = extents.get(query.getKind());
if (extent != null) {
// Make a copy of the list of all the entities in the extent
versionedEntities = extent.getAllEntities();
} else if (!query.hasKind()) {
// Kind-less query, so we need a list containing all entities of
// all kinds.
versionedEntities = profile.getAllEntities();
if (query.orderSize() == 0) {
// add a sort by key asc to match the behavior of prod
query.addOrder(new Order().setDirection(Query.Order.Direction.ASCENDING).setProperty(Entity.KEY_RESERVED_PROPERTY));
}
} else {
// no extent - we're querying for a kind without any entities
}
if (versionedEntities != null) {
queryEntities = new ArrayList<>();
versions = new HashMap<>();
for (VersionedEntity entity : versionedEntities) {
queryEntities.add(entity.entityProto());
versions.put(entity.entityProto().getKey(), entity.version());
}
}
}
// Give all entity groups with unapplied jobs the opportunity to catch
// up. Note that this will not impact the result of the query we're
// currently fulfilling since we already have the (unfiltered) result
// set.
profile.groom();
if (queryEntities == null) {
// so we don't need to check for null anywhere else down below
queryEntities = Collections.emptyList();
}
// Building filter predicate
List<Predicate<EntityProto>> predicates = new ArrayList<>();
// apply ancestor restriction
if (query.hasAncestor()) {
final List<Element> ancestorPath = query.getAncestor().getPath().elements();
predicates.add(new Predicate<EntityProto>() {
@Override
public boolean apply(EntityProto entity) {
List<Element> path = entity.getKey().getPath().elements();
return path.size() >= ancestorPath.size() && path.subList(0, ancestorPath.size()).equals(ancestorPath);
}
});
}
if (query.isShallow()) {
final long keyPathLength = query.hasAncestor() ? query.getAncestor().getPath().elementSize() + 1 : 1;
predicates.add(new Predicate<EntityProto>() {
@Override
public boolean apply(EntityProto entity) {
return entity.getKey().getPath().elementSize() == keyPathLength;
}
});
}
// apply namespace restriction
final boolean hasNamespace = query.hasNameSpace();
final String namespace = query.getNameSpace();
predicates.add(new Predicate<EntityProto>() {
@Override
public boolean apply(EntityProto entity) {
Reference ref = entity.getKey();
// Filter all elements not in the query's namespace.
if (hasNamespace) {
if (!ref.hasNameSpace() || !namespace.equals(ref.getNameSpace())) {
return false;
}
} else {
if (ref.hasNameSpace()) {
return false;
}
}
return true;
}
});
// Get entityComparator with filter matching capability
final EntityProtoComparator entityComparator = new EntityProtoComparator(validatedQuery.getQuery().orders(), validatedQuery.getQuery().filters());
// applying filter restrictions
predicates.add(new Predicate<EntityProto>() {
@Override
public boolean apply(EntityProto entity) {
return entityComparator.matches(entity);
}
});
Predicate<EntityProto> queryPredicate = Predicates.<EntityProto>not(Predicates.<EntityProto>and(predicates));
// The ordering of the following operations is important to maintain correct
// query functionality.
// Filtering entities
Iterables.removeIf(queryEntities, queryPredicate);
// Expanding projections
if (query.propertyNameSize() > 0) {
queryEntities = createIndexOnlyQueryResults(queryEntities, entityComparator);
}
// Sorting entities
Collections.sort(queryEntities, entityComparator);
// Apply group by. This must happen after sorting to select the correct first entity.
queryEntities = applyGroupByProperties(queryEntities, query);
// store the query and return the results
LiveQuery liveQuery = new LiveQuery(queryEntities, versions, query, entityComparator, clock);
// CompositeIndexManager does some filesystem reads/writes, so needs to
// be privileged.
AccessController.doPrivileged(new PrivilegedAction<Object>() {
@Override
public Object run() {
LocalCompositeIndexManager.getInstance().processQuery(validatedQuery.getV3Query());
return null;
}
});
// Using next function to prefetch results and return them from runQuery
QueryResult result = liveQuery.nextResult(query.hasOffset() ? query.getOffset() : null, query.hasCount() ? query.getCount() : null, query.isCompile());
if (query.isCompile()) {
result.setCompiledQuery(liveQuery.compileQuery());
}
if (result.isMoreResults()) {
long cursor = queryId.getAndIncrement();
profile.addQuery(cursor, liveQuery);
result.getMutableCursor().setApp(query.getApp()).setCursor(cursor);
}
// Copy the index list for the query into the result.
for (Index index : LocalCompositeIndexManager.getInstance().queryIndexList(query)) {
result.addIndex(wrapIndexInCompositeIndex(app, index));
}
// for
return result;
}
}
use of com.google.storage.onestore.v3.OnestoreEntity.Reference in project appengine-java-standard by GoogleCloudPlatform.
the class LocalDatastoreService method commitImpl.
/**
* Requires a lock on the provided profile.
*/
private CommitResponse commitImpl(LiveTxn liveTxn, final Profile profile) {
// assumes we already have a lock on the profile
CommitResponse response = new CommitResponse();
for (EntityGroupTracker tracker : liveTxn.getAllTrackers()) {
// This will throw an exception if the entity group
// has been modified since this transaction started.
tracker.checkEntityGroupVersion();
}
int deleted = 0;
int written = 0;
Cost totalCost = new Cost();
long commitTimestamp = profile.incrementAndGetCommitTimestamp();
for (EntityGroupTracker tracker : liveTxn.getAllTrackers()) {
Profile.EntityGroup eg = tracker.getEntityGroup();
eg.incrementVersion();
final Collection<EntityProto> writtenEntities = tracker.getWrittenEntities();
final Collection<Reference> deletedKeys = tracker.getDeletedKeys();
LocalDatastoreJob job = new WriteJob(highRepJobPolicy, eg, profile, commitTimestamp, writtenEntities, deletedKeys);
addTo(totalCost, job.calculateJobCost());
eg.addJob(job);
deleted += deletedKeys.size();
written += writtenEntities.size();
for (EntityProto writtenEntity : writtenEntities) {
response.addVersion().setRootEntityKey(writtenEntity.getKey()).setVersion(job.getMutationTimestamp(writtenEntity.getKey()));
}
for (Reference deletedKey : deletedKeys) {
response.addVersion().setRootEntityKey(deletedKey).setVersion(job.getMutationTimestamp(deletedKey));
}
}
logger.fine("committed: " + written + " puts, " + deleted + " deletes in " + liveTxn.getAllTrackers().size() + " entity groups");
response.setCost(totalCost);
return response;
}
use of com.google.storage.onestore.v3.OnestoreEntity.Reference in project appengine-java-standard by GoogleCloudPlatform.
the class LocalDatastoreService method deleteImpl.
// status
@SuppressWarnings("unused")
public DeleteResponse deleteImpl(Status status, DeleteRequest request) {
DeleteResponse response = new DeleteResponse();
if (request.keySize() == 0) {
return response;
}
Cost totalCost = response.getMutableCost();
// We don't support requests that span apps, so the app for the first key
// is the app for all keys.
String app = request.keys().get(0).getApp();
final Profile profile = getOrCreateProfile(app);
LiveTxn liveTxn = null;
// Maintain a mapping of keys by entity group so that we can apply one job
// per entity group.
Map<Path, List<Reference>> keysByEntityGroup = new LinkedHashMap<>();
Map<Reference, Long> writtenVersions = new HashMap<>();
synchronized (profile) {
for (final Reference key : request.keys()) {
validatePathComplete(key);
Path group = getGroup(key);
if (request.hasTransaction()) {
if (liveTxn == null) {
liveTxn = profile.getTxn(request.getTransaction().getHandle());
}
checkRequest(!liveTxn.isReadOnly(), "Cannot modify entities in a read-only transaction.");
Profile.EntityGroup eg = profile.getGroup(group);
// this will throw an exception if we attempt to modify
// the wrong entity group
eg.addTransaction(liveTxn).addDeletedEntity(key);
} else {
List<Reference> keysToDelete = keysByEntityGroup.get(group);
if (keysToDelete == null) {
keysToDelete = new ArrayList<>();
keysByEntityGroup.put(group, keysToDelete);
}
keysToDelete.add(key);
}
}
// does all the work for each entity group.
for (final Map.Entry<Path, List<Reference>> entry : keysByEntityGroup.entrySet()) {
Profile.EntityGroup eg = profile.getGroup(entry.getKey());
eg.incrementVersion();
LocalDatastoreJob job = new WriteJob(highRepJobPolicy, eg, profile, Collections.<EntityProto>emptyList(), entry.getValue());
addTo(totalCost, job.calculateJobCost());
eg.addJob(job);
for (Reference deletedKey : entry.getValue()) {
writtenVersions.put(deletedKey, job.getMutationTimestamp(deletedKey));
}
}
}
if (!request.hasTransaction()) {
for (Reference key : request.keys()) {
response.addVersion(writtenVersions.get(key));
}
}
return response;
}
Aggregations