Search in sources :

Example 1 with MapDocument

use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryMapsWithGroups.

@Test
public void queryMapsWithGroups() throws Exception {
    final List<String> textSamples = Arrays.asList(TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.ANGSTROM);
    final List<MapDocument> documents = IntStream.range(0, textSamples.size() / 2).mapToObj(i -> MapDocument.newBuilder().setDocId(i).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(textSamples.get(i * 2)).build()).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(textSamples.get(i * 2 + 1)).build()).setGroup(i % 2).build()).collect(Collectors.toList());
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_GROUPED_INDEX));
        documents.forEach(recordStore::saveRecord);
        assertEquals(Collections.singletonList(1L), queryMapDocumentsWithGroupedIndex("a", Query.field("value").text().containsPhrase("both alike in dignity"), 1L, 1376087127));
        assertEquals(Collections.singletonList(0L), queryMapDocumentsWithGroupedIndex("b", Query.field("value").text().containsAny("king anders"), 0L, -1204479544));
        commit(context);
    }
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) BY_GROUP(com.apple.foundationdb.record.IndexScanType.BY_GROUP) Matchers.not(org.hamcrest.Matchers.not) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) TextTokenizerFactory(com.apple.foundationdb.record.provider.common.text.TextTokenizerFactory) ComplexDocument(com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument) Subspace(com.apple.foundationdb.subspace.Subspace) TextSamples(com.apple.foundationdb.record.provider.common.text.TextSamples) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) COMPLEX_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC) VersionKeyExpression(com.apple.foundationdb.record.metadata.expressions.VersionKeyExpression) Map(java.util.Map) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) PlanMatchers.textComparison(com.apple.foundationdb.record.query.plan.match.PlanMatchers.textComparison) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) Arguments(org.junit.jupiter.params.provider.Arguments) BY_VALUE(com.apple.foundationdb.record.IndexScanType.BY_VALUE) TupleRange(com.apple.foundationdb.record.TupleRange) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.anything(org.hamcrest.Matchers.anything) Matchers.contains(org.hamcrest.Matchers.contains) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) PlanMatchers.typeFilter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.typeFilter) Matchers.containsString(org.hamcrest.Matchers.containsString) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) ComponentWithComparison(com.apple.foundationdb.record.query.expressions.ComponentWithComparison) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) PlanMatchers(com.apple.foundationdb.record.query.plan.match.PlanMatchers) BunchedMap(com.apple.foundationdb.map.BunchedMap) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) LoggableException(com.apple.foundationdb.util.LoggableException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nullable(javax.annotation.Nullable) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) SOURCE_EXHAUSTED(com.apple.foundationdb.record.RecordCursor.NoNextReason.SOURCE_EXHAUSTED) Tags(com.apple.test.Tags) SCAN_LIMIT_REACHED(com.apple.foundationdb.record.RecordCursor.NoNextReason.SCAN_LIMIT_REACHED) OrComponent(com.apple.foundationdb.record.query.expressions.OrComponent) BunchedMapScanEntry(com.apple.foundationdb.map.BunchedMapScanEntry) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) AndOrComponent(com.apple.foundationdb.record.query.expressions.AndOrComponent) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) Matcher(org.hamcrest.Matcher) PlanMatchers.unorderedUnion(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unorderedUnion) TextIndexBunchedSerializerTest.entryOf(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexBunchedSerializerTest.entryOf) IndexEntry(com.apple.foundationdb.record.IndexEntry) PlanMatchers.groupingBounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.groupingBounds) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) LoggerFactory(org.slf4j.LoggerFactory) BY_RANK(com.apple.foundationdb.record.IndexScanType.BY_RANK) PrefixTextTokenizer(com.apple.foundationdb.record.provider.common.text.PrefixTextTokenizer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) SubspaceSplitter(com.apple.foundationdb.map.SubspaceSplitter) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Tuple(com.apple.foundationdb.tuple.Tuple) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PlanMatchers.textIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.textIndexScan) TextTokenizerRegistryImpl(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistryImpl) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) RETURN_LIMIT_REACHED(com.apple.foundationdb.record.RecordCursor.NoNextReason.RETURN_LIMIT_REACHED) FDBExceptions(com.apple.foundationdb.record.provider.foundationdb.FDBExceptions) MethodSource(org.junit.jupiter.params.provider.MethodSource) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) ImmutableSet(com.google.common.collect.ImmutableSet) KeyValue(com.apple.foundationdb.KeyValue) SimpleDocument(com.apple.foundationdb.record.TestRecordsTextProto.SimpleDocument) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ImmutableMap(com.google.common.collect.ImmutableMap) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.anyOf(org.hamcrest.Matchers.anyOf) IntStream(java.util.stream.IntStream) PlanMatchers.primaryKeyDistinct(com.apple.foundationdb.record.query.plan.match.PlanMatchers.primaryKeyDistinct) Descriptors(com.google.protobuf.Descriptors) CompletableFuture(java.util.concurrent.CompletableFuture) BooleanNormalizer(com.apple.foundationdb.record.query.plan.planning.BooleanNormalizer) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) DefaultTextTokenizer(com.apple.foundationdb.record.provider.common.text.DefaultTextTokenizer) BY_TEXT_TOKEN(com.apple.foundationdb.record.IndexScanType.BY_TEXT_TOKEN) BunchedMapMultiIterator(com.apple.foundationdb.map.BunchedMapMultiIterator) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) SIMPLE_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) BY_TIME_WINDOW(com.apple.foundationdb.record.IndexScanType.BY_TIME_WINDOW) FilteringTextTokenizer(com.apple.foundationdb.record.provider.common.text.FilteringTextTokenizer) ReadTransaction(com.apple.foundationdb.ReadTransaction) Normalizer(java.text.Normalizer) Matchers.any(org.hamcrest.Matchers.any) TimeUnit(java.util.concurrent.TimeUnit) DefaultTextTokenizerFactory(com.apple.foundationdb.record.provider.common.text.DefaultTextTokenizerFactory) PlanMatchers.unbounded(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unbounded) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Comparator(java.util.Comparator) Collections(java.util.Collections) AllSuffixesTextTokenizer(com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer) MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Matchers.containsString(org.hamcrest.Matchers.containsString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 2 with MapDocument

use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.

the class TextIndexTest method queryMapDocuments.

@Test
public void queryMapDocuments() throws Exception {
    final List<String> textSamples = Arrays.asList(TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.ANGSTROM, TextSamples.AETHELRED, TextSamples.FRENCH);
    final List<MapDocument> documents = IntStream.range(0, textSamples.size() / 2).mapToObj(i -> MapDocument.newBuilder().setDocId(i).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(textSamples.get(i * 2)).build()).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(textSamples.get(i * 2 + 1)).build()).setGroup(i % 2).build()).collect(Collectors.toList());
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX));
        documents.forEach(recordStore::saveRecord);
        assertEquals(Collections.singletonList(2L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAny("king unknown_token"), 1059912699, true));
        assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034960, true));
        assertEquals(Collections.emptyList(), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034991, true));
        assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPrefix("na"), 1125182095, true));
        assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAllPrefixes("civ mut ha"), 0, false));
        assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsAnyPrefix("civ mut na"), 0, true));
        RecordQuery queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.and(Query.field("group").equalsValue(0L), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsAny("anders king"))))).build();
        RecordQueryPlan planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
        assertThat(planWithAdditionalFilter, filter(Query.field("group").equalsValue(0L), descendant(textIndexScan(anything()))));
        List<Long> queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
        assertEquals(Collections.singletonList(0L), queryResults);
        queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.or(Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("a"), Query.field("value").text().containsPhrase("bury their parents strife"))), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsPrefix("th"))))).build();
        planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
        assertThat(planWithAdditionalFilter, primaryKeyDistinct(unorderedUnion(descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))), descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))))));
        queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
        assertEquals(3, queryResults.size());
        assertEquals(ImmutableSet.of(0L, 1L, 2L), ImmutableSet.copyOf(queryResults));
        // Planner bug that can happen with certain malformed queries. This plan actually
        // returns records where the key and the value match in the same entry, but it is
        // asking for all records where *any* entry has a key matching "a" and *any* entry
        // has a value matching the text predicate. In reality, this is probably a sign
        // the user didn't input their query correctly, but it requires more work from the
        // planner not to plan this kind of query.
        // FIXME: Full Text: The Planner doesn't always correctly handle ands with nesteds (https://github.com/FoundationDB/fdb-record-layer/issues/53)
        final QueryComponent malformedMapFilter = Query.and(Query.field("entry").oneOfThem().matches(Query.field("key").equalsValue("a")), Query.field("entry").oneOfThem().matches(Query.field("value").text().containsAll("civil hands unclean")));
        RecordQueryPlan malformedMapPlan = planner.plan(RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(malformedMapFilter).build());
        assertThat(malformedMapPlan, descendant(textIndexScan(allOf(indexName(MAP_ON_VALUE_INDEX.getName()), groupingBounds(allOf(notNullValue(), hasTupleString("[[a],[a]]"))), textComparison(equalTo(new Comparisons.TextComparison(Comparisons.Type.TEXT_CONTAINS_ALL, "civil hands unclean", null, DefaultTextTokenizer.NAME)))))));
        commit(context);
    }
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) BY_GROUP(com.apple.foundationdb.record.IndexScanType.BY_GROUP) Matchers.not(org.hamcrest.Matchers.not) MetaDataException(com.apple.foundationdb.record.metadata.MetaDataException) TextTokenizerFactory(com.apple.foundationdb.record.provider.common.text.TextTokenizerFactory) ComplexDocument(com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument) Subspace(com.apple.foundationdb.subspace.Subspace) TextSamples(com.apple.foundationdb.record.provider.common.text.TextSamples) RecordCursorResult(com.apple.foundationdb.record.RecordCursorResult) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) COMPLEX_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC) VersionKeyExpression(com.apple.foundationdb.record.metadata.expressions.VersionKeyExpression) Map(java.util.Map) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) Query(com.apple.foundationdb.record.query.expressions.Query) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Matchers.allOf(org.hamcrest.Matchers.allOf) Set(java.util.Set) PlanMatchers.textComparison(com.apple.foundationdb.record.query.plan.match.PlanMatchers.textComparison) FanType(com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType) Arguments(org.junit.jupiter.params.provider.Arguments) BY_VALUE(com.apple.foundationdb.record.IndexScanType.BY_VALUE) TupleRange(com.apple.foundationdb.record.TupleRange) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) Stream(java.util.stream.Stream) PlanMatchers.indexName(com.apple.foundationdb.record.query.plan.match.PlanMatchers.indexName) Matchers.anything(org.hamcrest.Matchers.anything) Matchers.contains(org.hamcrest.Matchers.contains) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) PlanMatchers.typeFilter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.typeFilter) Matchers.containsString(org.hamcrest.Matchers.containsString) FDBIndexedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) PlanMatchers.fetch(com.apple.foundationdb.record.query.plan.match.PlanMatchers.fetch) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) ComponentWithComparison(com.apple.foundationdb.record.query.expressions.ComponentWithComparison) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) ArrayList(java.util.ArrayList) PlanMatchers(com.apple.foundationdb.record.query.plan.match.PlanMatchers) BunchedMap(com.apple.foundationdb.map.BunchedMap) TestLogMessageKeys(com.apple.foundationdb.record.logging.TestLogMessageKeys) LoggableException(com.apple.foundationdb.util.LoggableException) Matchers.lessThan(org.hamcrest.Matchers.lessThan) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nullable(javax.annotation.Nullable) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) FieldWithComparison(com.apple.foundationdb.record.query.expressions.FieldWithComparison) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) SOURCE_EXHAUSTED(com.apple.foundationdb.record.RecordCursor.NoNextReason.SOURCE_EXHAUSTED) Tags(com.apple.test.Tags) SCAN_LIMIT_REACHED(com.apple.foundationdb.record.RecordCursor.NoNextReason.SCAN_LIMIT_REACHED) OrComponent(com.apple.foundationdb.record.query.expressions.OrComponent) BunchedMapScanEntry(com.apple.foundationdb.map.BunchedMapScanEntry) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) AndOrComponent(com.apple.foundationdb.record.query.expressions.AndOrComponent) Comparisons(com.apple.foundationdb.record.query.expressions.Comparisons) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Index(com.apple.foundationdb.record.metadata.Index) Matcher(org.hamcrest.Matcher) PlanMatchers.unorderedUnion(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unorderedUnion) TextIndexBunchedSerializerTest.entryOf(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexBunchedSerializerTest.entryOf) IndexEntry(com.apple.foundationdb.record.IndexEntry) PlanMatchers.groupingBounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.groupingBounds) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) LoggerFactory(org.slf4j.LoggerFactory) BY_RANK(com.apple.foundationdb.record.IndexScanType.BY_RANK) PrefixTextTokenizer(com.apple.foundationdb.record.provider.common.text.PrefixTextTokenizer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) SubspaceSplitter(com.apple.foundationdb.map.SubspaceSplitter) PlanMatchers.bounds(com.apple.foundationdb.record.query.plan.match.PlanMatchers.bounds) RecordQueryPlanner(com.apple.foundationdb.record.query.plan.RecordQueryPlanner) Tuple(com.apple.foundationdb.tuple.Tuple) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) PlanMatchers.textIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.textIndexScan) TextTokenizerRegistryImpl(com.apple.foundationdb.record.provider.common.text.TextTokenizerRegistryImpl) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) RETURN_LIMIT_REACHED(com.apple.foundationdb.record.RecordCursor.NoNextReason.RETURN_LIMIT_REACHED) FDBExceptions(com.apple.foundationdb.record.provider.foundationdb.FDBExceptions) MethodSource(org.junit.jupiter.params.provider.MethodSource) PlanMatchers.coveringIndexScan(com.apple.foundationdb.record.query.plan.match.PlanMatchers.coveringIndexScan) ImmutableSet(com.google.common.collect.ImmutableSet) KeyValue(com.apple.foundationdb.KeyValue) SimpleDocument(com.apple.foundationdb.record.TestRecordsTextProto.SimpleDocument) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ImmutableMap(com.google.common.collect.ImmutableMap) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) TextTokenizer(com.apple.foundationdb.record.provider.common.text.TextTokenizer) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) List(java.util.List) EvaluationContext(com.apple.foundationdb.record.EvaluationContext) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) Matchers.equalTo(org.hamcrest.Matchers.equalTo) MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Matchers.anyOf(org.hamcrest.Matchers.anyOf) IntStream(java.util.stream.IntStream) PlanMatchers.primaryKeyDistinct(com.apple.foundationdb.record.query.plan.match.PlanMatchers.primaryKeyDistinct) Descriptors(com.google.protobuf.Descriptors) CompletableFuture(java.util.concurrent.CompletableFuture) BooleanNormalizer(com.apple.foundationdb.record.query.plan.planning.BooleanNormalizer) PlanHashable(com.apple.foundationdb.record.PlanHashable) PlanMatchers.filter(com.apple.foundationdb.record.query.plan.match.PlanMatchers.filter) HashSet(java.util.HashSet) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) ScanProperties(com.apple.foundationdb.record.ScanProperties) RecordCursorIterator(com.apple.foundationdb.record.RecordCursorIterator) DefaultTextTokenizer(com.apple.foundationdb.record.provider.common.text.DefaultTextTokenizer) BY_TEXT_TOKEN(com.apple.foundationdb.record.IndexScanType.BY_TEXT_TOKEN) BunchedMapMultiIterator(com.apple.foundationdb.map.BunchedMapMultiIterator) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) EmptyKeyExpression(com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression) SIMPLE_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) RecordTypeBuilder(com.apple.foundationdb.record.metadata.RecordTypeBuilder) BY_TIME_WINDOW(com.apple.foundationdb.record.IndexScanType.BY_TIME_WINDOW) FilteringTextTokenizer(com.apple.foundationdb.record.provider.common.text.FilteringTextTokenizer) ReadTransaction(com.apple.foundationdb.ReadTransaction) Normalizer(java.text.Normalizer) Matchers.any(org.hamcrest.Matchers.any) TimeUnit(java.util.concurrent.TimeUnit) DefaultTextTokenizerFactory(com.apple.foundationdb.record.provider.common.text.DefaultTextTokenizerFactory) PlanMatchers.unbounded(com.apple.foundationdb.record.query.plan.match.PlanMatchers.unbounded) FDBDatabaseFactory(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory) Message(com.google.protobuf.Message) RecordCursor(com.apple.foundationdb.record.RecordCursor) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) PlanMatchers.descendant(com.apple.foundationdb.record.query.plan.match.PlanMatchers.descendant) Comparator(java.util.Comparator) Collections(java.util.Collections) AllSuffixesTextTokenizer(com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer) RecordQueryPlan(com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) QueryComponent(com.apple.foundationdb.record.query.expressions.QueryComponent) Matchers.containsString(org.hamcrest.Matchers.containsString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) FDBQueriedRecord(com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) RecordQuery(com.apple.foundationdb.record.query.RecordQuery) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 3 with MapDocument

use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.

the class TextIndexTest method saveMapDocuments.

@Test
public void saveMapDocuments() throws Exception {
    final MapDocument firstDocument = MapDocument.newBuilder().setDocId(1066L).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(TextSamples.ANGSTROM)).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(TextSamples.ROMEO_AND_JULIET_PROLOGUE)).build();
    final MapDocument secondDocument = MapDocument.newBuilder().setDocId(1415L).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(TextSamples.OLD_S)).addEntry(MapDocument.Entry.newBuilder().setKey("c").setValue(TextSamples.FRENCH)).build();
    final MapDocument documentWithEmpties = MapDocument.newBuilder().setDocId(1815L).addEntry(MapDocument.Entry.newBuilder().setKey(// value intentionally left unset
    "a")).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue("")).build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> {
            metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
            metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX);
        });
        recordStore.saveRecord(firstDocument);
        recordStore.saveRecord(secondDocument);
        recordStore.saveRecord(documentWithEmpties);
        assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(0)), entryOf(Tuple.from(1415L), Collections.singletonList(7))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("a", "the")));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Arrays.asList(22, 25))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("b", "civil")));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), Collections.singletonList(4))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("c", "a")));
        commit(context);
    }
}
Also used : MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 4 with MapDocument

use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.

the class TextIndexTest method tokenizerVersionChangeWithMultipleEntries.

@Test
public void tokenizerVersionChangeWithMultipleEntries() throws Exception {
    final MapDocument map1 = MapDocument.newBuilder().setDocId(1066L).addEntry(MapDocument.Entry.newBuilder().setKey("fr").setValue(TextSamples.FRENCH)).addEntry(MapDocument.Entry.newBuilder().setKey("de").setValue(TextSamples.GERMAN)).build();
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> {
            metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
            metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_PREFIX_LEGACY);
        });
        recordStore.saveRecord(map1);
        List<Map.Entry<Tuple, List<Integer>>> scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX_LEGACY, Tuple.from("fr", "rec"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(5))), scannedEntries);
        scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX_LEGACY, Tuple.from("de", "wah"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(8))), scannedEntries);
        assertEquals(Collections.singletonList(Pair.of(Tuple.from(1066L), 0)), scanTokenizerVersions(recordStore, MAP_ON_VALUE_PREFIX_LEGACY));
        commit(context);
    }
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> {
            metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
            metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_PREFIX);
        });
        // save an document where most of the source entries remain the same but are re-indexed because of the tokenizer version change
        final MapDocument map1prime = map1.toBuilder().addEntry(MapDocument.Entry.newBuilder().setKey("yi").setValue(TextSamples.YIDDISH)).build();
        recordStore.saveRecord(map1prime);
        List<Map.Entry<Tuple, List<Integer>>> scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX, Tuple.from("yi", "שפרא"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(1))), scannedEntries);
        scannedEntries = toMapEntries(scanIndex(recordStore, MAP_ON_VALUE_PREFIX, TupleRange.prefixedBy("rec").prepend(Tuple.from("fr"))), Tuple.from("fr"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from("recu", 1066L), Collections.singletonList(5))), scannedEntries);
        scannedEntries = toMapEntries(scanIndex(recordStore, MAP_ON_VALUE_PREFIX, TupleRange.prefixedBy("wah").prepend(Tuple.from("de"))), Tuple.from("de"));
        assertEquals(Collections.singletonList(entryOf(Tuple.from("wahr", 1066L), Collections.singletonList(8))), scannedEntries);
        assertEquals(Collections.singletonList(Pair.of(Tuple.from(1066L), 1)), scanTokenizerVersions(recordStore, MAP_ON_VALUE_PREFIX_LEGACY));
        commit(context);
    }
}
Also used : MapDocument(com.apple.foundationdb.record.TestRecordsTextProto.MapDocument) BunchedMapScanEntry(com.apple.foundationdb.map.BunchedMapScanEntry) IndexEntry(com.apple.foundationdb.record.IndexEntry) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

MapDocument (com.apple.foundationdb.record.TestRecordsTextProto.MapDocument)4 BunchedMapScanEntry (com.apple.foundationdb.map.BunchedMapScanEntry)3 IndexEntry (com.apple.foundationdb.record.IndexEntry)3 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)3 Test (org.junit.jupiter.api.Test)3 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)3 KeyValue (com.apple.foundationdb.KeyValue)2 ReadTransaction (com.apple.foundationdb.ReadTransaction)2 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)2 BunchedMap (com.apple.foundationdb.map.BunchedMap)2 BunchedMapMultiIterator (com.apple.foundationdb.map.BunchedMapMultiIterator)2 SubspaceSplitter (com.apple.foundationdb.map.SubspaceSplitter)2 EvaluationContext (com.apple.foundationdb.record.EvaluationContext)2 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)2 BY_GROUP (com.apple.foundationdb.record.IndexScanType.BY_GROUP)2 BY_RANK (com.apple.foundationdb.record.IndexScanType.BY_RANK)2 BY_TEXT_TOKEN (com.apple.foundationdb.record.IndexScanType.BY_TEXT_TOKEN)2 BY_TIME_WINDOW (com.apple.foundationdb.record.IndexScanType.BY_TIME_WINDOW)2 BY_VALUE (com.apple.foundationdb.record.IndexScanType.BY_VALUE)2 PlanHashable (com.apple.foundationdb.record.PlanHashable)2