Search in sources :

Example 1 with IndexQueryabilityFilter

use of com.apple.foundationdb.record.query.IndexQueryabilityFilter in project fdb-record-layer by FoundationDB.

the class FDBRecordStore method getSnapshotRecordCount.

@Override
public CompletableFuture<Long> getSnapshotRecordCount(@Nonnull KeyExpression key, @Nonnull Key.Evaluated value, @Nonnull IndexQueryabilityFilter indexQueryabilityFilter) {
    if (getRecordMetaData().getRecordCountKey() != null) {
        if (key.getColumnSize() != value.size()) {
            throw recordCoreException("key and value are not the same size");
        }
        final ReadTransaction tr = context.readTransaction(true);
        final Tuple subkey = Tuple.from(RECORD_COUNT_KEY).addAll(value.toTupleAppropriateList());
        if (getRecordMetaData().getRecordCountKey().equals(key)) {
            return tr.get(getSubspace().pack(subkey)).thenApply(FDBRecordStore::decodeRecordCount);
        } else if (key.isPrefixKey(getRecordMetaData().getRecordCountKey())) {
            AsyncIterable<KeyValue> kvs = tr.getRange(getSubspace().range(Tuple.from(RECORD_COUNT_KEY)));
            return MoreAsyncUtil.reduce(getExecutor(), kvs.iterator(), 0L, (count, kv) -> count + decodeRecordCount(kv.getValue()));
        }
    }
    return evaluateAggregateFunction(Collections.emptyList(), IndexFunctionHelper.count(key), TupleRange.allOf(value.toTuple()), IsolationLevel.SNAPSHOT, indexQueryabilityFilter).thenApply(tuple -> tuple.getLong(0));
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) UnaryOperator(java.util.function.UnaryOperator) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) RecordSerializer(com.apple.foundationdb.record.provider.common.RecordSerializer) Subspace(com.apple.foundationdb.subspace.Subspace) MutationType(com.apple.foundationdb.MutationType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Map(java.util.Map) RecordIndexUniquenessViolation(com.apple.foundationdb.record.RecordIndexUniquenessViolation) QueryToKeyMatcher(com.apple.foundationdb.record.query.QueryToKeyMatcher) InvalidProtocolBufferException(com.google.protobuf.InvalidProtocolBufferException) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) Set(java.util.Set) TupleRange(com.apple.foundationdb.record.TupleRange) KeySpacePath(com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath) ByteOrder(java.nio.ByteOrder) SyntheticRecordType(com.apple.foundationdb.record.metadata.SyntheticRecordType) RecordMetaDataProvider(com.apple.foundationdb.record.RecordMetaDataProvider) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) API(com.apple.foundationdb.annotation.API) FunctionNames(com.apple.foundationdb.record.FunctionNames) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Supplier(java.util.function.Supplier) FormerIndex(com.apple.foundationdb.record.metadata.FormerIndex) ArrayList(java.util.ArrayList) ByteScanLimiter(com.apple.foundationdb.record.ByteScanLimiter) ParameterRelationshipGraph(com.apple.foundationdb.record.query.ParameterRelationshipGraph) LoggableException(com.apple.foundationdb.util.LoggableException) CloseableAsyncIterator(com.apple.foundationdb.async.CloseableAsyncIterator) IndexRecordFunction(com.apple.foundationdb.record.metadata.IndexRecordFunction) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) CursorLimitManager(com.apple.foundationdb.record.cursors.CursorLimitManager) ExecuteState(com.apple.foundationdb.record.ExecuteState) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) DynamicMessageRecordSerializer(com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer) SyntheticRecordPlanner(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner) IndexEntry(com.apple.foundationdb.record.IndexEntry) LoggerFactory(org.slf4j.LoggerFactory) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ByteBuffer(java.nio.ByteBuffer) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Range(com.apple.foundationdb.Range) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PipelineOperation(com.apple.foundationdb.record.PipelineOperation) RecordMetaDataProto(com.apple.foundationdb.record.RecordMetaDataProto) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) AndComponent(com.apple.foundationdb.record.query.expressions.AndComponent) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) RecordTypeKeyComparison(com.apple.foundationdb.record.query.expressions.RecordTypeKeyComparison) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Optional(java.util.Optional) MutableRecordStoreState(com.apple.foundationdb.record.MutableRecordStoreState) RecordTypeOrBuilder(com.apple.foundationdb.record.metadata.RecordTypeOrBuilder) SyntheticRecordFromStoredRecordPlan(com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordFromStoredRecordPlan) SpotBugsSuppressWarnings(com.apple.foundationdb.annotation.SpotBugsSuppressWarnings) Descriptors(com.google.protobuf.Descriptors) AsyncIterator(com.apple.foundationdb.async.AsyncIterator) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) CursorStreamingMode(com.apple.foundationdb.record.CursorStreamingMode) Key(com.apple.foundationdb.record.metadata.Key) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) EndpointType(com.apple.foundationdb.record.EndpointType) ScanProperties(com.apple.foundationdb.record.ScanProperties) Suppliers(com.google.common.base.Suppliers) LinkedList(java.util.LinkedList) Nonnull(javax.annotation.Nonnull) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) IndexState(com.apple.foundationdb.record.IndexState) StoreRecordFunction(com.apple.foundationdb.record.metadata.StoreRecordFunction) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) FDBRecordStoreStateCache(com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) ReadTransaction(com.apple.foundationdb.ReadTransaction) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) Tuple(com.apple.foundationdb.tuple.Tuple)

Example 2 with IndexQueryabilityFilter

use of com.apple.foundationdb.record.query.IndexQueryabilityFilter in project fdb-record-layer by FoundationDB.

the class FDBRestrictedIndexQueryTest method indexQueryabilityFilter.

/**
 * Verify that the query planner uses the specified {@link com.apple.foundationdb.record.query.IndexQueryabilityFilter}.
 * If both allowed indexes and a queryability filter are set, verify that the planner uses the allowed indexes.
 */
@DualPlannerTest
void indexQueryabilityFilter() {
    RecordMetaDataHook hook = metaData -> {
        metaData.removeIndex("MySimpleRecord$str_value_indexed");
        metaData.addIndex("MySimpleRecord", new Index("limited_str_value_index", field("str_value_indexed"), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.NOT_ALLOWED_FOR_QUERY_OPTIONS));
    };
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context, hook);
        recordStore.deleteAllRecords();
        commit(context);
    }
    RecordQuery query1 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("abc")).build();
    // Scan(<,>) | [MySimpleRecord] | str_value_indexed EQUALS abc
    // Scan(<,>) | [MySimpleRecord] | $5eb5afb5-7c31-4fa4-bd7a-cec3027b6ade/str_value_indexed EQUALS abc
    RecordQueryPlan plan1 = planner.plan(query1);
    assertThat("should not use prohibited index", plan1, hasNoDescendant(indexScan("limited_str_value_index")));
    assertTrue(plan1.hasFullRecordScan(), "should use full record scan");
    if (planner instanceof RecordQueryPlanner) {
        assertEquals(-223683738, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    // TODO: Issue https://github.com/FoundationDB/fdb-record-layer/issues/1074
    // assertEquals(-1148834070, plan1.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    } else {
        assertEquals(-2136471589, plan1.planHash(PlanHashable.PlanHashKind.LEGACY));
    }
    RecordQuery query2 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("abc")).setIndexQueryabilityFilter(IndexQueryabilityFilter.TRUE).build();
    // Index(limited_str_value_index [[abc],[abc]])
    RecordQueryPlan plan2 = planner.plan(query2);
    assertThat("explicitly use any index", plan2, descendant(indexScan("limited_str_value_index")));
    assertFalse(plan2.hasRecordScan(), "should not use record scan");
    assertEquals(-1573180774, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(994464666, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1531627068, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
    RecordQuery query3 = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("str_value_indexed").equalsValue("abc")).setIndexQueryabilityFilter(IndexQueryabilityFilter.FALSE).setAllowedIndex("limited_str_value_index").build();
    // Index(limited_str_value_index [[abc],[abc]])
    RecordQueryPlan plan3 = planner.plan(query3);
    assertThat("should use allowed index despite index queryability filter", plan3, descendant(indexScan("limited_str_value_index")));
    assertFalse(plan3.hasRecordScan(), "should not use record scan");
    assertEquals(-1573180774, plan2.planHash(PlanHashable.PlanHashKind.LEGACY));
    assertEquals(994464666, plan2.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
    assertEquals(-1531627068, plan2.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
}
Also used : LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Tuple(com.apple.foundationdb.tuple.Tuple) TestHelpers(com.apple.foundationdb.record.TestHelpers) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Predicate(java.util.function.Predicate) Matchers.allOf(org.hamcrest.Matchers.allOf) IndexQueryabilityFilter(com.apple.foundationdb.record.query.IndexQueryabilityFilter) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) Objects(java.util.Objects) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) RecordStoreState(com.apple.foundationdb.record.RecordStoreState) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) PlanMatchers.hasNoDescendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasNoDescendant) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) AggregateFunctionNotSupportedException(com.apple.foundationdb.record.AggregateFunctionNotSupportedException) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) FunctionNames(com.apple.foundationdb.record.FunctionNames) IndexAggregateFunction(com.apple.foundationdb.record.metadata.IndexAggregateFunction) PlanMatchers.indexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexScan) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) CompletableFuture(java.util.concurrent.CompletableFuture) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) RangeSet(com.apple.foundationdb.async.RangeSet) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) Function(java.util.function.Function) PlanHashable(com.apple.foundationdb.record.PlanHashable) Key(com.apple.foundationdb.record.metadata.Key) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) IsolationLevel(com.apple.foundationdb.record.IsolationLevel) Tags(com.apple.test.Tags) Index(com.apple.foundationdb.record.metadata.Index) Executable(org.junit.jupiter.api.function.Executable) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Collections(java.util.Collections) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Index(com.apple.foundationdb.record.metadata.Index) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) RecordQuery(com.apple.foundationdb.record.query.RecordQuery)

Aggregations

RangeSet (com.apple.foundationdb.async.RangeSet)2 AggregateFunctionNotSupportedException (com.apple.foundationdb.record.AggregateFunctionNotSupportedException)2 FunctionNames (com.apple.foundationdb.record.FunctionNames)2 IsolationLevel (com.apple.foundationdb.record.IsolationLevel)2 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)2 RecordCursor (com.apple.foundationdb.record.RecordCursor)2 RecordStoreState (com.apple.foundationdb.record.RecordStoreState)2 TupleRange (com.apple.foundationdb.record.TupleRange)2 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)2 Index (com.apple.foundationdb.record.metadata.Index)2 IndexAggregateFunction (com.apple.foundationdb.record.metadata.IndexAggregateFunction)2 IndexTypes (com.apple.foundationdb.record.metadata.IndexTypes)2 Key (com.apple.foundationdb.record.metadata.Key)2 EmptyKeyExpression (com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression)2 IndexQueryabilityFilter (com.apple.foundationdb.record.query.IndexQueryabilityFilter)2 RecordQuery (com.apple.foundationdb.record.query.RecordQuery)2 Query (com.apple.foundationdb.record.query.expressions.Query)2 RecordQueryPlanner (com.apple.foundationdb.record.query.plan.RecordQueryPlanner)2 RecordQueryPlan (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan)2 Tuple (com.apple.foundationdb.tuple.Tuple)2