Examples with Index com.google.storage.onestore.v3.OnestoreEntity.Index used on opensource projects

Example 1 with Index

use of com.google.storage.onestore.v3.OnestoreEntity.Index in project appengine-java-standard by GoogleCloudPlatform.

the class AsyncDatastoreServiceImpl method doBatchPut.

@Override
protected Future<List<Key>> doBatchPut(@Nullable Transaction txn, final List<Entity> entities) {
    PutRequest baseReq = new PutRequest();
    if (txn != null) {
        TransactionImpl.ensureTxnActive(txn);
        baseReq.setTransaction(InternalTransactionV3.toProto(txn));
    }
    // Do not group when inside a transaction.
    boolean group = !baseReq.hasTransaction();
    final List<Integer> order = Lists.newArrayListWithCapacity(entities.size());
    Iterator<PutRequest> batches = putBatcher.getBatches(entities, baseReq, baseReq.getSerializedSize(), group, order);
    List<Future<PutResponse>> futures = putBatcher.makeCalls(batches);
    return registerInTransaction(txn, new ReorderingMultiFuture<PutResponse, List<Key>>(futures, order) {

        @Override
        protected List<Key> aggregate(PutResponse intermediateResult, Iterator<Integer> indexItr, List<Key> result) {
            for (Reference reference : intermediateResult.keys()) {
                int index = indexItr.next();
                Key key = entities.get(index).getKey();
                KeyTranslator.updateKey(reference, key);
                result.set(index, key);
            }
            return result;
        }

        @Override
        protected List<Key> initResult() {
            // Create an array pre-populated with null values (twice :-))
            List<Key> result = new ArrayList<Key>(Collections.<Key>nCopies(order.size(), null));
            return result;
        }
    });
}

Example 2 with Index

use of com.google.storage.onestore.v3.OnestoreEntity.Index in project appengine-java-standard by GoogleCloudPlatform.

the class CompositeIndexManager method compositeIndexForQuery.

/**
 * Given a {@link IndexComponentsOnlyQuery}, return the {@link Index} needed to fulfill the query,
 * or {@code null} if no index is needed.
 *
 * <p>This code needs to remain in sync with its counterparts in other languages. If you modify
 * this code please make sure you make the same update in the local datastore for other languages.
 *
 * @param indexOnlyQuery The query.
 * @return The index that must be present in order to fulfill the query, or {@code null} if no
 *     index is needed.
 */
@Nullable
protected Index compositeIndexForQuery(final IndexComponentsOnlyQuery indexOnlyQuery) {
    DatastoreV3Pb.Query query = indexOnlyQuery.getQuery();
    boolean hasKind = query.hasKind();
    boolean isAncestor = query.hasAncestor();
    List<Filter> filters = query.filters();
    List<Order> orders = query.orders();
    if (filters.isEmpty() && orders.isEmpty()) {
        // built-in primary key or kind index can handle this.
        return null;
    }
    // Group the filters by operator.
    List<String> eqProps = indexOnlyQuery.getPrefix();
    List<Property> indexProperties = indexOnlyQuery.isGeo() ? getNeededSearchProps(eqProps, indexOnlyQuery.getGeoProperties()) : getRecommendedIndexProps(indexOnlyQuery);
    if (hasKind && !eqProps.isEmpty() && eqProps.size() == filters.size() && !indexOnlyQuery.hasKeyProperty() && orders.isEmpty()) {
        // specified, and those queries can _not_ be satisfied by merge-join.
        return null;
    }
    if (hasKind && !isAncestor && indexProperties.size() <= 1 && !indexOnlyQuery.isGeo() && (!indexOnlyQuery.hasKeyProperty() || indexProperties.get(0).getDirectionEnum() == Property.Direction.ASCENDING)) {
        // (a.k.a. Search) indexes, we might.)
        return null;
    }
    Index index = new Index();
    index.setEntityType(query.getKind());
    index.setAncestor(isAncestor);
    index.mutablePropertys().addAll(indexProperties);
    return index;
}

Example 3 with Index

use of com.google.storage.onestore.v3.OnestoreEntity.Index in project appengine-java-standard by GoogleCloudPlatform.

the class CompositeIndexManager method minimumCompositeIndexForQuery.

/**
 * Given a {@link IndexComponentsOnlyQuery} and a collection of existing {@link Index}s, return
 * the minimum {@link Index} needed to fulfill the query, or {@code null} if no index is needed.
 *
 * <p>This code needs to remain in sync with its counterparts in other languages. If you modify
 * this code please make sure you make the same update in the local datastore for other languages.
 *
 * @param indexOnlyQuery The query.
 * @param indexes The existing indexes.
 * @return The minimum index that must be present in order to fulfill the query, or {@code null}
 *     if no index is needed.
 */
@Nullable
protected Index minimumCompositeIndexForQuery(IndexComponentsOnlyQuery indexOnlyQuery, Collection<Index> indexes) {
    Index suggestedIndex = compositeIndexForQuery(indexOnlyQuery);
    if (suggestedIndex == null) {
        return null;
    }
    if (indexOnlyQuery.isGeo()) {
        // None of the shortcuts/optimizations below are applicable for Search indexes.
        return suggestedIndex;
    }
    class EqPropsAndAncestorConstraint {

        final Set<String> equalityProperties;

        final boolean ancestorConstraint;

        EqPropsAndAncestorConstraint(Set<String> equalityProperties, boolean ancestorConstraint) {
            this.equalityProperties = equalityProperties;
            this.ancestorConstraint = ancestorConstraint;
        }
    }
    // Map from postfix to the remaining equality properties and ancestor constraints.
    Map<List<Property>, EqPropsAndAncestorConstraint> remainingMap = new HashMap<List<Property>, EqPropsAndAncestorConstraint>();
    index_for: for (Index index : indexes) {
        if (// Kind must match.
        !indexOnlyQuery.getQuery().getKind().equals(index.getEntityType()) || // Ancestor indexes can only be used on ancestor queries.
        (!indexOnlyQuery.getQuery().hasAncestor() && index.isAncestor())) {
            continue;
        }
        // Matching the postfix.
        int postfixSplit = index.propertySize();
        for (IndexComponent component : Lists.reverse(indexOnlyQuery.getPostfix())) {
            if (!component.matches(index.propertys().subList(Math.max(postfixSplit - component.size(), 0), postfixSplit))) {
                continue index_for;
            }
            postfixSplit -= component.size();
        }
        // Postfix matches! Now checking the prefix.
        Set<String> indexEqProps = Sets.newHashSetWithExpectedSize(postfixSplit);
        for (Property prop : index.propertys().subList(0, postfixSplit)) {
            // Index must not contain extra properties in the prefix.
            if (!indexOnlyQuery.getPrefix().contains(prop.getName())) {
                continue index_for;
            }
            indexEqProps.add(prop.getName());
        }
        // Index matches!
        // Find the matching remaining requirements.
        List<Property> indexPostfix = index.propertys().subList(postfixSplit, index.propertySize());
        Set<String> remainingEqProps;
        boolean remainingAncestor;
        {
            EqPropsAndAncestorConstraint remaining = remainingMap.get(indexPostfix);
            if (remaining == null) {
                remainingEqProps = Sets.newHashSet(indexOnlyQuery.getPrefix());
                remainingAncestor = indexOnlyQuery.getQuery().hasAncestor();
            } else {
                remainingEqProps = remaining.equalityProperties;
                remainingAncestor = remaining.ancestorConstraint;
            }
        }
        // Remove any remaining requirements handled by this index.
        boolean modified = remainingEqProps.removeAll(indexEqProps);
        if (remainingAncestor && index.isAncestor()) {
            modified = true;
            remainingAncestor = false;
        }
        if (remainingEqProps.isEmpty() && !remainingAncestor) {
            // No new index needed!
            return null;
        }
        if (!modified) {
            // Index made no contribution, don't update the map.
            continue;
        }
        // Save indexes contribution
        remainingMap.put(indexPostfix, new EqPropsAndAncestorConstraint(remainingEqProps, remainingAncestor));
    }
    if (remainingMap.isEmpty()) {
        // suggested index is the minimum index
        return suggestedIndex;
    }
    int minimumCost = Integer.MAX_VALUE;
    List<Property> minimumPostfix = null;
    EqPropsAndAncestorConstraint minimumRemaining = null;
    for (Map.Entry<List<Property>, EqPropsAndAncestorConstraint> entry : remainingMap.entrySet()) {
        int cost = entry.getValue().equalityProperties.size();
        if (entry.getValue().ancestorConstraint) {
            // Arbitrary value picked because ancestor are multi-valued.
            cost += 2;
        }
        if (cost < minimumCost) {
            minimumCost = cost;
            minimumPostfix = entry.getKey();
            minimumRemaining = entry.getValue();
        }
    }
    // map not empty so we should have found cost < MAX_VALUE.
    requireNonNull(minimumRemaining);
    requireNonNull(minimumPostfix);
    // Populating suggesting the minimal index instead.
    suggestedIndex.clearProperty();
    suggestedIndex.setAncestor(minimumRemaining.ancestorConstraint);
    for (String name : minimumRemaining.equalityProperties) {
        suggestedIndex.addProperty().setName(name).setDirection(Direction.ASCENDING);
    }
    Collections.sort(suggestedIndex.mutablePropertys(), PROPERTY_NAME_COMPARATOR);
    suggestedIndex.mutablePropertys().addAll(minimumPostfix);
    return suggestedIndex;
}

Example 4 with Index

use of com.google.storage.onestore.v3.OnestoreEntity.Index in project appengine-java-standard by GoogleCloudPlatform.

the class CompositeIndexUtils method generateXmlForIndex.

/**
 * Generate an xml representation of the provided {@link Index}.
 *
 * <pre>{@code
 * <datastore-indexes autoGenerate="true">
 *     <datastore-index kind="a" ancestor="false">
 *         <property name="yam" direction="asc"/>
 *         <property name="not yam" direction="desc"/>
 *     </datastore-index>
 * </datastore-indexes>
 * }</pre>
 *
 * @param index The index for which we want an xml representation.
 * @param source The source of the provided index.
 * @return The xml representation of the provided index.
 */
public static String generateXmlForIndex(Index index, IndexSource source) {
    // Careful with pb method names - index.hasAncestor just returns whether or not
    // the ancestor was set, not the value of the field itself.  We always provide
    // a value for this field, so we just want the value.
    boolean isAncestor = index.isAncestor();
    if (index.propertySize() == 0) {
        return String.format(DATASTORE_INDEX_NO_PROPERTIES_XML_FORMAT, index.getEntityType(), isAncestor, source);
    }
    boolean isSearchIndex = false;
    StringBuilder sb = new StringBuilder();
    for (Property prop : index.propertys()) {
        String extraAttribute;
        if (prop.getDirectionEnum() == Direction.ASCENDING) {
            extraAttribute = ASC_ATTRIBUTE;
        } else if (prop.getDirectionEnum() == Direction.DESCENDING) {
            extraAttribute = DESC_ATTRIBUTE;
        } else if (prop.getModeEnum() == Mode.GEOSPATIAL) {
            isSearchIndex = true;
            extraAttribute = GEOSPATIAL_ATTRIBUTE;
        } else {
            extraAttribute = "";
        }
        sb.append(String.format(PROPERTY_XML_FORMAT, prop.getName(), extraAttribute));
    }
    String ancestorAttribute = isSearchIndex ? "" : String.format(ANCESTOR_ATTRIBUTE_FORMAT, isAncestor);
    return String.format(DATASTORE_INDEX_WITH_PROPERTIES_XML_FORMAT, index.getEntityType(), ancestorAttribute, source, sb.toString());
}

Example 5 with Index

use of com.google.storage.onestore.v3.OnestoreEntity.Index in project appengine-java-standard by GoogleCloudPlatform.

the class IndexComponentsOnlyQuery method categorizeQuery.

private void categorizeQuery() {
    Set<String> ineqProps = Sets.newHashSet();
    hasKeyProperty = false;
    for (Filter filter : query.filters()) {
        String propName = filter.getProperty(0).getName();
        switch(filter.getOpEnum()) {
            case EQUAL:
                equalityProps.add(propName);
                break;
            case EXISTS:
                existsProps.add(propName);
                break;
            case GREATER_THAN:
            case GREATER_THAN_OR_EQUAL:
            case LESS_THAN:
            case LESS_THAN_OR_EQUAL:
                ineqProps.add(propName);
                break;
            case CONTAINED_IN_REGION:
                containmentProps.add(propName);
                break;
            default:
                throw new IllegalArgumentException("Unable to categorize query using filter operator " + filter.getOp());
        }
        if (propName.equals(Entity.KEY_RESERVED_PROPERTY)) {
            hasKeyProperty = true;
        }
    }
    // Add the inequality filter properties, if any.
    if (query.orderSize() == 0 && !ineqProps.isEmpty()) {
        // We do not add an index property for the inequality filter because
        // it will be taken care of when we add the sort on that same property
        // down below.
        orderProps.add(new Property().setName(ineqProps.iterator().next()));
    }
    groupByProps.addAll(query.groupByPropertyNames());
    // If a property is included in the group by, its existance will be satisfied.
    existsProps.removeAll(groupByProps);
    // Add orders.
    for (Order order : query.orders()) {
        if (order.getProperty().equals(Entity.KEY_RESERVED_PROPERTY)) {
            hasKeyProperty = true;
        }
        // If a property is in the ordering, it has already been satisfied.
        groupByProps.remove(order.getProperty());
        orderProps.add(new Property().setName(order.getProperty()).setDirection(order.getDirection()));
    }
}