Examples with Order com.google.apphosting.datastore.DatastoreV3Pb.Query.Order used on opensource projects

Example 11 with Order

use of com.google.apphosting.datastore.DatastoreV3Pb.Query.Order in project appengine-java-standard by GoogleCloudPlatform.

the class ValidatedQuery method validateQuery.

/**
 * Determines if a given query is supported by the datastore.
 *
 * @throws IllegalQueryException If the provided query fails validation.
 */
private void validateQuery() {
    if (query.propertyNameSize() > 0 && query.isKeysOnly()) {
        throw new IllegalQueryException("projection and keys_only cannot both be set", IllegalQueryType.ILLEGAL_PROJECTION);
    }
    Set<String> projectionProperties = new HashSet<String>(query.propertyNameSize());
    for (String property : query.propertyNames()) {
        if (!projectionProperties.add(property)) {
            throw new IllegalQueryException("cannot project a property multiple times", IllegalQueryType.ILLEGAL_PROJECTION);
        }
    }
    Set<String> groupBySet = Sets.newHashSetWithExpectedSize(query.groupByPropertyNameSize());
    for (String name : query.groupByPropertyNames()) {
        if (!groupBySet.add(name)) {
            throw new IllegalQueryException("cannot group by a property multiple times", IllegalQueryType.ILLEGAL_GROUPBY);
        }
        // filters.
        if (Entity.RESERVED_NAME.matcher(name).matches()) {
            throw new IllegalQueryException("group by is not supported for the property: " + name, IllegalQueryType.ILLEGAL_GROUPBY);
        }
    }
    /* Validate group by properties in orderings. */
    Set<String> groupByInOrderSet = Sets.newHashSetWithExpectedSize(query.groupByPropertyNameSize());
    for (Order order : query.orders()) {
        if (groupBySet.contains(order.getProperty())) {
            groupByInOrderSet.add(order.getProperty());
        } else if (groupByInOrderSet.size() != groupBySet.size()) {
            throw new IllegalQueryException("must specify all group by orderings before any non group by orderings", IllegalQueryType.ILLEGAL_GROUPBY);
        }
    }
    // Transaction requires ancestor
    if (query.hasTransaction() && !query.hasAncestor()) {
        throw new IllegalQueryException("Only ancestor queries are allowed inside transactions.", IllegalQueryType.TRANSACTION_REQUIRES_ANCESTOR);
    }
    // Filters and sort orders require kind.
    if (!query.hasKind()) {
        for (Filter filter : query.filters()) {
            if (!filter.getProperty(0).getName().equals(Entity.KEY_RESERVED_PROPERTY)) {
                throw new IllegalQueryException("kind is required for non-__key__ filters", IllegalQueryType.KIND_REQUIRED);
            }
        }
        for (Order order : query.orders()) {
            if (!(order.getProperty().equals(Entity.KEY_RESERVED_PROPERTY) && order.getDirection() == Order.Direction.ASCENDING.getValue())) {
                throw new IllegalQueryException("kind is required for all orders except __key__ ascending", IllegalQueryType.KIND_REQUIRED);
            }
        }
    }
    /* Validate ancestor, if it exists. */
    if (query.hasAncestor()) {
        Reference ancestor = query.getAncestor();
        if (!ancestor.getApp().equals(query.getApp())) {
            throw new IllegalQueryException("The query app is " + query.getApp() + " but ancestor app is " + ancestor.getApp(), IllegalQueryType.ILLEGAL_VALUE);
        }
        if (!ancestor.getNameSpace().equals(query.getNameSpace())) {
            throw new IllegalQueryException("The query namespace is " + query.getNameSpace() + " but ancestor namespace is " + ancestor.getNameSpace(), IllegalQueryType.ILLEGAL_VALUE);
        }
    }
    // Check for unsupported filter values.  We only support one property
    // per filter and one property with an inequality filter.
    String ineqProp = null;
    this.isGeo = false;
    for (Filter filter : query.filters()) {
        int numProps = filter.propertySize();
        if (numProps != 1) {
            throw new IllegalQueryException(String.format("Filter has %s properties, expected 1", numProps), IllegalQueryType.FILTER_WITH_MULTIPLE_PROPS);
        }
        Property prop = filter.getProperty(0);
        String propName = prop.getName();
        /* Extra validation for __key__. The filter value must be a key,
       * if the query has a kind, the key's kind must match, and the
       * app and namespace must match the query. */
        if (Entity.KEY_RESERVED_PROPERTY.equals(propName)) {
            PropertyValue value = prop.getValue();
            if (!value.hasReferenceValue()) {
                throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter value must be a Key", IllegalQueryType.ILLEGAL_VALUE);
            }
            ReferenceValue refVal = value.getReferenceValue();
            if (!refVal.getApp().equals(query.getApp())) {
                throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter app is " + refVal.getApp() + " but query app is " + query.getApp(), IllegalQueryType.ILLEGAL_VALUE);
            }
            if (!refVal.getNameSpace().equals(query.getNameSpace())) {
                throw new IllegalQueryException(Entity.KEY_RESERVED_PROPERTY + " filter namespace is " + refVal.getNameSpace() + " but query namespace is " + query.getNameSpace(), IllegalQueryType.ILLEGAL_VALUE);
            }
        }
        if (INEQUALITY_OPERATORS.contains(filter.getOpEnum())) {
            if (ineqProp == null) {
                ineqProp = propName;
            } else if (!ineqProp.equals(propName)) {
                throw new IllegalQueryException(String.format("Only one inequality filter per query is supported.  " + "Encountered both %s and %s", ineqProp, propName), IllegalQueryType.MULTIPLE_INEQ_FILTERS);
            }
        } else if (filter.getOpEnum() == Operator.EQUAL) {
            if (projectionProperties.contains(propName)) {
                throw new IllegalQueryException("cannot use projection on a property with an equality filter", IllegalQueryType.ILLEGAL_PROJECTION);
            } else if (groupBySet.contains(propName)) {
                throw new IllegalQueryException("cannot use group by on a property with an equality filter", IllegalQueryType.ILLEGAL_GROUPBY);
            }
        } else if (filter.getOpEnum() == Operator.CONTAINED_IN_REGION) {
            isGeo = true;
            if (!filter.hasGeoRegion() || prop.getValue().hasPointValue()) {
                throw new IllegalQueryException(String.format("Geo-spatial filter on %s should specify GeoRegion rather than Property Value", propName), IllegalQueryType.UNSUPPORTED_FILTER);
            }
            GeoRegion region = filter.getGeoRegion();
            if ((region.hasCircle() && region.hasRectangle()) || (!region.hasCircle() && !region.hasRectangle())) {
                throw new IllegalQueryException(String.format("Geo-spatial filter on %s should specify Circle or Rectangle, but not both", propName), IllegalQueryType.UNSUPPORTED_FILTER);
            }
        } else if (UNSUPPORTED_OPERATORS.contains(filter.getOpEnum())) {
            throw new IllegalQueryException(String.format("Unsupported filter operator: %s", filter.getOp()), IllegalQueryType.UNSUPPORTED_FILTER);
        }
    }
    if (isGeo) {
        if (ineqProp != null) {
            throw new IllegalQueryException("Inequality filter with geo-spatial query is not supported.", IllegalQueryType.UNSUPPORTED_FILTER);
        }
        if (query.hasAncestor()) {
            throw new IllegalQueryException("Geo-spatial filter on ancestor query is not supported.", IllegalQueryType.UNSUPPORTED_FILTER);
        }
        if (query.hasCompiledCursor() || query.hasEndCompiledCursor()) {
            throw new IllegalQueryException("Start and end cursors are not supported on geo-spatial queries.", IllegalQueryType.CURSOR_NOT_SUPPORTED);
        }
    }
    if (ineqProp != null && query.groupByPropertyNameSize() > 0) {
        if (!groupBySet.contains(ineqProp)) {
            throw new IllegalQueryException(String.format("Inequality filter on %s must also be a group by property when " + "group by properties are set.", ineqProp), IllegalQueryType.ILLEGAL_GROUPBY);
        }
    }
    if (ineqProp != null) {
        if (query.orderSize() > 0) {
            if (!ineqProp.equals(query.getOrder(0).getProperty())) {
                // First order must match the inequality filter.
                throw new IllegalQueryException(String.format("The first sort property must be the same as the property to which " + "the inequality filter is applied.  In your query the first sort property " + "is %s but the inequality filter is on %s", query.getOrder(0).getProperty(), ineqProp), IllegalQueryType.FIRST_SORT_NEQ_INEQ_PROP);
            }
        }
    }
}

Example 12 with Order

use of com.google.apphosting.datastore.DatastoreV3Pb.Query.Order in project appengine-java-standard by GoogleCloudPlatform.

the class NormalizedQuery method normalizeQuery.

private void normalizeQuery() {
    // TODO: consider sharing this code with MegastoreQueryPlanner
    // NOTE: Keep in sync with MegastoreQueryPlanner#normalize()
    Set<Property> equalityFilterProperties = new HashSet<Property>();
    Set<String> equalityProperties = new HashSet<String>();
    Set<String> inequalityProperties = new HashSet<String>();
    for (Iterator<Filter> itr = query.mutableFilters().iterator(); itr.hasNext(); ) {
        Filter f = itr.next();
        /* Normalize IN filters to EQUAL. */
        if (f.propertySize() == 1 && f.getOpEnum() == Operator.IN) {
            f.setOp(Operator.EQUAL);
        }
        if (f.propertySize() >= 1) {
            String name = f.getProperty(0).getName();
            if (f.getOpEnum() == Operator.EQUAL) {
                if (!equalityFilterProperties.add(f.getProperty(0))) {
                    // The filter is an exact duplicate, remove it.
                    itr.remove();
                } else {
                    equalityProperties.add(name);
                }
            } else if (INEQUALITY_OPERATORS.contains(f.getOpEnum())) {
                inequalityProperties.add(name);
            }
        }
    }
    equalityProperties.removeAll(inequalityProperties);
    // Strip repeated orders and orders coinciding with EQUAL filters.
    for (Iterator<Order> itr = query.mutableOrders().iterator(); itr.hasNext(); ) {
        if (!equalityProperties.add(itr.next().getProperty())) {
            itr.remove();
        }
    }
    Set<String> allProperties = equalityProperties;
    allProperties.addAll(inequalityProperties);
    // Removing redundant exists filters.
    for (Iterator<Filter> itr = query.mutableFilters().iterator(); itr.hasNext(); ) {
        Filter f = itr.next();
        if (f.getOpEnum() == Operator.EXISTS && f.propertySize() >= 1 && !allProperties.add(f.getProperty(0).getName())) {
            itr.remove();
        }
    }
    // Adding exist filters for any requested properties or group by properties that need them.
    for (String propName : Iterables.concat(query.propertyNames(), query.groupByPropertyNames())) {
        if (allProperties.add(propName)) {
            query.addFilter().setOp(Operator.EXISTS).addProperty().setName(propName).setMultiple(false).getMutableValue();
        }
    }
    /* Strip all orders if filtering on __key__ with equals. */
    for (Filter f : query.filters()) {
        if (f.getOpEnum() == Operator.EQUAL && f.propertySize() >= 1 && f.getProperty(0).getName().equals(Entity.KEY_RESERVED_PROPERTY)) {
            query.clearOrder();
            break;
        }
    }
    /* Strip all orders that follow a ordering on __key__ as keys are unique
     * thus additional ordering has no effect. */
    boolean foundKeyOrder = false;
    for (Iterator<Order> i = query.mutableOrders().iterator(); i.hasNext(); ) {
        String property = i.next().getProperty();
        if (foundKeyOrder) {
            i.remove();
        } else if (property.equals(Entity.KEY_RESERVED_PROPERTY)) {
            foundKeyOrder = true;
        }
    }
}

Example 13 with Order

use of com.google.apphosting.datastore.DatastoreV3Pb.Query.Order in project appengine-java-standard by GoogleCloudPlatform.

the class LocalDatastoreService method runQuery.

// status
@SuppressWarnings("unused")
public QueryResult runQuery(Status status, Query query) {
    // Construct a validated query right away so we can fail fast
    // if something is wrong.
    final LocalCompositeIndexManager.ValidatedQuery validatedQuery = new LocalCompositeIndexManager.ValidatedQuery(query);
    query = validatedQuery.getV3Query();
    // LocalCompositeIndexManager.ValidatedQuery.  I don't know why.
    try {
        CursorModernizer.modernizeQueryCursors(query);
    } catch (InvalidConversionException e) {
        throw newError(ErrorCode.BAD_REQUEST, "Invalid cursor");
    }
    String app = query.getApp();
    Profile profile = getOrCreateProfile(app);
    synchronized (profile) {
        // ancestor does not imply we have a transaction.
        if (query.hasTransaction() || query.hasAncestor()) {
            // Query can only have a txn if it is an ancestor query.  Either way we
            // know we've got an ancestor.
            Path groupPath = getGroup(query.getAncestor());
            Profile.EntityGroup eg = profile.getGroup(groupPath);
            if (query.hasTransaction()) {
                if (!app.equals(query.getTransaction().getApp())) {
                    throw newError(ErrorCode.INTERNAL_ERROR, "Can't query app " + app + "in a transaction on app " + query.getTransaction().getApp());
                }
                LiveTxn liveTxn = profile.getTxn(query.getTransaction().getHandle());
                // this will throw an exception if we attempt to read from
                // the wrong entity group
                eg.addTransaction(liveTxn);
                // Use snapshot profile.
                profile = eg.getSnapshot(liveTxn);
            }
            if (query.hasAncestor()) {
                if (query.hasTransaction() || !query.hasFailoverMs()) {
                    // Either we have a transaction or the user has requested strongly
                    // consistent results.  Either way, we need to apply jobs.
                    eg.rollForwardUnappliedJobs();
                }
            }
        }
        if (query.hasSearchQuery()) {
            throw newError(ErrorCode.BAD_REQUEST, "full-text search unsupported");
        }
        // Run as a PseudoKind query if necessary, otherwise check the actual local datastore
        List<EntityProto> queryEntities = pseudoKinds.runQuery(query);
        Map<Reference, Long> versions = null;
        if (queryEntities == null) {
            Collection<VersionedEntity> versionedEntities = null;
            Map<String, Extent> extents = profile.getExtents();
            Extent extent = extents.get(query.getKind());
            if (extent != null) {
                // Make a copy of the list of all the entities in the extent
                versionedEntities = extent.getAllEntities();
            } else if (!query.hasKind()) {
                // Kind-less query, so we need a list containing all entities of
                // all kinds.
                versionedEntities = profile.getAllEntities();
                if (query.orderSize() == 0) {
                    // add a sort by key asc to match the behavior of prod
                    query.addOrder(new Order().setDirection(Query.Order.Direction.ASCENDING).setProperty(Entity.KEY_RESERVED_PROPERTY));
                }
            } else {
            // no extent - we're querying for a kind without any entities
            }
            if (versionedEntities != null) {
                queryEntities = new ArrayList<>();
                versions = new HashMap<>();
                for (VersionedEntity entity : versionedEntities) {
                    queryEntities.add(entity.entityProto());
                    versions.put(entity.entityProto().getKey(), entity.version());
                }
            }
        }
        // Give all entity groups with unapplied jobs the opportunity to catch
        // up.  Note that this will not impact the result of the query we're
        // currently fulfilling since we already have the (unfiltered) result
        // set.
        profile.groom();
        if (queryEntities == null) {
            // so we don't need to check for null anywhere else down below
            queryEntities = Collections.emptyList();
        }
        // Building filter predicate
        List<Predicate<EntityProto>> predicates = new ArrayList<>();
        // apply ancestor restriction
        if (query.hasAncestor()) {
            final List<Element> ancestorPath = query.getAncestor().getPath().elements();
            predicates.add(new Predicate<EntityProto>() {

                @Override
                public boolean apply(EntityProto entity) {
                    List<Element> path = entity.getKey().getPath().elements();
                    return path.size() >= ancestorPath.size() && path.subList(0, ancestorPath.size()).equals(ancestorPath);
                }
            });
        }
        if (query.isShallow()) {
            final long keyPathLength = query.hasAncestor() ? query.getAncestor().getPath().elementSize() + 1 : 1;
            predicates.add(new Predicate<EntityProto>() {

                @Override
                public boolean apply(EntityProto entity) {
                    return entity.getKey().getPath().elementSize() == keyPathLength;
                }
            });
        }
        // apply namespace restriction
        final boolean hasNamespace = query.hasNameSpace();
        final String namespace = query.getNameSpace();
        predicates.add(new Predicate<EntityProto>() {

            @Override
            public boolean apply(EntityProto entity) {
                Reference ref = entity.getKey();
                // Filter all elements not in the query's namespace.
                if (hasNamespace) {
                    if (!ref.hasNameSpace() || !namespace.equals(ref.getNameSpace())) {
                        return false;
                    }
                } else {
                    if (ref.hasNameSpace()) {
                        return false;
                    }
                }
                return true;
            }
        });
        // Get entityComparator with filter matching capability
        final EntityProtoComparator entityComparator = new EntityProtoComparator(validatedQuery.getQuery().orders(), validatedQuery.getQuery().filters());
        // applying filter restrictions
        predicates.add(new Predicate<EntityProto>() {

            @Override
            public boolean apply(EntityProto entity) {
                return entityComparator.matches(entity);
            }
        });
        Predicate<EntityProto> queryPredicate = Predicates.<EntityProto>not(Predicates.<EntityProto>and(predicates));
        // The ordering of the following operations is important to maintain correct
        // query functionality.
        // Filtering entities
        Iterables.removeIf(queryEntities, queryPredicate);
        // Expanding projections
        if (query.propertyNameSize() > 0) {
            queryEntities = createIndexOnlyQueryResults(queryEntities, entityComparator);
        }
        // Sorting entities
        Collections.sort(queryEntities, entityComparator);
        // Apply group by. This must happen after sorting to select the correct first entity.
        queryEntities = applyGroupByProperties(queryEntities, query);
        // store the query and return the results
        LiveQuery liveQuery = new LiveQuery(queryEntities, versions, query, entityComparator, clock);
        // CompositeIndexManager does some filesystem reads/writes, so needs to
        // be privileged.
        AccessController.doPrivileged(new PrivilegedAction<Object>() {

            @Override
            public Object run() {
                LocalCompositeIndexManager.getInstance().processQuery(validatedQuery.getV3Query());
                return null;
            }
        });
        // Using next function to prefetch results and return them from runQuery
        QueryResult result = liveQuery.nextResult(query.hasOffset() ? query.getOffset() : null, query.hasCount() ? query.getCount() : null, query.isCompile());
        if (query.isCompile()) {
            result.setCompiledQuery(liveQuery.compileQuery());
        }
        if (result.isMoreResults()) {
            long cursor = queryId.getAndIncrement();
            profile.addQuery(cursor, liveQuery);
            result.getMutableCursor().setApp(query.getApp()).setCursor(cursor);
        }
        // Copy the index list for the query into the result.
        for (Index index : LocalCompositeIndexManager.getInstance().queryIndexList(query)) {
            result.addIndex(wrapIndexInCompositeIndex(app, index));
        }
        // for
        return result;
    }
}

Example 14 with Order

use of com.google.apphosting.datastore.DatastoreV3Pb.Query.Order in project appengine-java-standard by GoogleCloudPlatform.

the class KeyFilteredPseudoKind method runQuery.

@Override
public List<EntityProto> runQuery(Query query) {
    Key startKey = null;
    Key endKey = null;
    boolean startInclusive = false;
    boolean endInclusive = false;
    /* We could leave all the filters and orders to LocalDatastoreService, but that would force
     * queries to examine all entities even when filters are present, and wouldn't
     * report the invalid-query errors of the real datastore.
     *
     * Explicit code to handle range, as {@link com.google.appengine.api.datastore.FilterMatcher}
     * is somewhat specialized to deal with {@link PropertyValue} in {@link EntityProto}s.
     */
    for (Filter filter : query.filters()) {
        Operator op = filter.getOpEnum();
        // Report error for filters we can't handle
        checkRequest(filter.propertySize() == 1 && filter.getProperty(0).getName().equals(Entity.KEY_RESERVED_PROPERTY) && (op == Operator.LESS_THAN || op == Operator.LESS_THAN_OR_EQUAL || op == Operator.GREATER_THAN || op == Operator.GREATER_THAN_OR_EQUAL || op == Operator.EQUAL), "Only comparison filters on " + Entity.KEY_RESERVED_PROPERTY + " supported");
        Object filterVal = DataTypeTranslator.getPropertyValue(filter.getProperty(0));
        // Redundant with {@link com.google.com.appengine.api.datastore.ValidatedQuery}
        checkRequest(filterVal instanceof Key, Entity.KEY_RESERVED_PROPERTY + " must be compared to a key");
        Key keyLimit = (Key) filterVal;
        // Update our search limits based on the filters
        if (op == Operator.LESS_THAN) {
            if (endKey == null || keyLimit.compareTo(endKey) <= 0) {
                endKey = keyLimit;
                endInclusive = false;
            }
        } else if (op == Operator.LESS_THAN_OR_EQUAL || op == Operator.EQUAL) {
            if (endKey == null || keyLimit.compareTo(endKey) < 0) {
                endKey = keyLimit;
                endInclusive = true;
            }
        }
        if (op == Operator.GREATER_THAN) {
            if (startKey == null || keyLimit.compareTo(startKey) >= 0) {
                startKey = keyLimit;
                startInclusive = false;
            }
        } else if (op == Operator.GREATER_THAN_OR_EQUAL || op == Operator.EQUAL) {
            if (startKey == null || keyLimit.compareTo(startKey) > 0) {
                startKey = keyLimit;
                startInclusive = true;
            }
        }
    }
    query.clearFilter();
    // The only allowed order we can handle is an initial ascending on key
    if (query.orderSize() > 0) {
        Order order = query.getOrder(0);
        if (order.getDirectionEnum() == Order.Direction.ASCENDING && Entity.KEY_RESERVED_PROPERTY.equals(order.getProperty())) {
            query.removeOrder(0);
        }
    }
    checkRequest(query.orderSize() == 0, "Only ascending order on " + Entity.KEY_RESERVED_PROPERTY + " supported");
    return runQuery(query, startKey, startInclusive, endKey, endInclusive);
}

Example 15 with Order

use of com.google.apphosting.datastore.DatastoreV3Pb.Query.Order in project appengine-java-standard by GoogleCloudPlatform.

the class EntityComparatorTests method testDefaultSort.

@Test
public void testDefaultSort() {
    // if no sorts provided we end up with the default sort
    List<Order> orders = Lists.newArrayList();
    List<? extends BaseEntityComparator<?>> entityComps = Arrays.asList(new EntityProtoComparator(orders), new EntityComparator(orders));
    for (BaseEntityComparator<?> comp : entityComps) {
        assertThat(comp.orders).containsExactly(KEY_ASC_ORDER);
    }
    // if a sort a non-key field is provided we add the default sort to the
    // end.
    Order notAKeySort = new Order().setProperty("not a key");
    orders.add(notAKeySort);
    entityComps = Arrays.asList(new EntityProtoComparator(orders), new EntityComparator(orders));
    for (BaseEntityComparator<?> comp : entityComps) {
        assertThat(comp.orders).containsExactly(notAKeySort, KEY_ASC_ORDER).inOrder();
    }
    // if we explicitly add the default sort to the end we don't end up with
    // more than one default sort
    orders.add(KEY_ASC_ORDER);
    entityComps = Arrays.asList(new EntityProtoComparator(orders), new EntityComparator(orders));
    for (BaseEntityComparator<?> comp : entityComps) {
        assertThat(comp.orders).containsExactly(notAKeySort, KEY_ASC_ORDER).inOrder();
    }
}