Search in sources :

Example 56 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method searchForAutoCompleteCrossingMultipleFields.

@Test
void searchForAutoCompleteCrossingMultipleFields() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context, metaDataBuilder -> {
            metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
            metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE);
        });
        // Write 8 texts and 6 of them contain the key "good"
        recordStore.saveRecord(createComplexDocument(1623L, "Good morning", "", 1));
        recordStore.saveRecord(createComplexDocument(1624L, "Good afternoon", "", 1));
        recordStore.saveRecord(createComplexDocument(1625L, "good evening", "", 1));
        recordStore.saveRecord(createComplexDocument(1626L, "Good night", "", 1));
        recordStore.saveRecord(createComplexDocument(1627L, "", "That's really good!", 1));
        recordStore.saveRecord(createComplexDocument(1628L, "", "I'm good", 1));
        recordStore.saveRecord(createComplexDocument(1629L, "", "Hello Record Layer", 1));
        RecordType recordType = recordStore.saveRecord(createComplexDocument(1630L, "", "Hello FoundationDB!", 1)).getRecordType();
        List<IndexEntry> results = recordStore.scanIndex(COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from("good")), null, ScanProperties.FORWARD_SCAN).asList().get();
        results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
        assertEquals(6, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
        assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(COMPLEX_MULTIPLE_TEXT_INDEXES_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
        // Assert the count of suggestions
        assertEquals(6, results.size());
        // Assert the suggestions' keys
        List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
        assertEquals(ImmutableList.of("good evening", "Good night", "Good morning", "Good afternoon", "I'm good", "That's really good!"), suggestions);
        // Assert the corresponding field for the suggestions
        List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
        assertEquals(ImmutableList.of("text", "text", "text", "text", "text2", "text2"), fields);
        // Assert the suggestions are sorted according to their values, which are determined by the position of the term into the indexed text
        List<Long> values = results.stream().map(i -> (Long) i.getValue().get(0)).collect(Collectors.toList());
        List<Long> valuesSorted = new ArrayList<>(values);
        Collections.sort(valuesSorted, Collections.reverseOrder());
        assertEquals(valuesSorted, values);
        assertEquals(values.get(0), values.get(1));
        assertEquals(values.get(1), values.get(2));
        assertEquals(values.get(2), values.get(3));
        assertTrue(values.get(3) > values.get(4));
        assertTrue(values.get(4) > values.get(5));
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) Arrays(java.util.Arrays) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) SynonymAnalyzer(com.apple.foundationdb.record.lucene.synonym.SynonymAnalyzer) EnglishSynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.EnglishSynonymMapConfig) Subspace(com.apple.foundationdb.subspace.Subspace) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) COMPLEX_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) SynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.SynonymMapConfig) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) FDBRecordContextConfig(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContextConfig) IsCollectionWithSize.hasSize(org.hamcrest.collection.IsCollectionWithSize.hasSize) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) List(java.util.List) NgramAnalyzer(com.apple.foundationdb.record.lucene.ngram.NgramAnalyzer) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) FDBDirectory(com.apple.foundationdb.record.lucene.directory.FDBDirectory) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Expressions.function(com.apple.foundationdb.record.metadata.Key.Expressions.function) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordLayerPropertyStorage(com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage) Descriptors(com.google.protobuf.Descriptors) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) IndexKeyValueToPartialRecord(com.apple.foundationdb.record.query.plan.IndexKeyValueToPartialRecord) ImmutableList(com.google.common.collect.ImmutableList) ScanProperties(com.apple.foundationdb.record.ScanProperties) FDBLuceneFileReference(com.apple.foundationdb.record.lucene.directory.FDBLuceneFileReference) TextIndexTestUtils(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Nullable(javax.annotation.Nullable) SIMPLE_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) SequenceInputStream(java.io.SequenceInputStream) IndexFileNames(org.apache.lucene.index.IndexFileNames) IOException(java.io.IOException) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) AutoService(com.google.auto.service.AutoService) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) Collections(java.util.Collections) AllSuffixesTextTokenizer(com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer) InputStream(java.io.InputStream) RecordType(com.apple.foundationdb.record.metadata.RecordType) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) ArrayList(java.util.ArrayList) IndexEntry(com.apple.foundationdb.record.IndexEntry) Test(org.junit.jupiter.api.Test)

Example 57 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method simpleInsertAndSearchNumFDBFetches.

@Test
void simpleInsertAndSearchNumFDBFetches() {
    try (FDBRecordContext context = openContext()) {
        rebuildIndexMetaData(context, SIMPLE_DOC, SIMPLE_TEXT_SUFFIXES);
        recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
        recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
        RecordCursor<IndexEntry> indexEntries = recordStore.scanIndex(SIMPLE_TEXT_SUFFIXES, IndexScanType.BY_LUCENE_FULL_TEXT, TupleRange.allOf(Tuple.from("Vision")), null, ScanProperties.FORWARD_SCAN);
        assertEquals(1, recordStore.scanIndex(SIMPLE_TEXT_SUFFIXES, IndexScanType.BY_LUCENE_FULL_TEXT, TupleRange.allOf(Tuple.from("Vision")), null, ScanProperties.FORWARD_SCAN).getCount().join());
        assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LOAD_SCAN_ENTRY).getCount());
        assertCorrectMetricCount(FDBStoreTimer.Events.LUCENE_GET_FILE_REFERENCE, 1);
        assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(SIMPLE_TEXT_SUFFIXES), context, "_0.cfs", true);
    }
}
Also used : FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) IndexEntry(com.apple.foundationdb.record.IndexEntry) Test(org.junit.jupiter.api.Test)

Example 58 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method testGroupedRecordSearch.

@Test
void testGroupedRecordSearch() {
    try (FDBRecordContext context = openContext()) {
        rebuildIndexMetaData(context, MAP_DOC, MAP_ON_VALUE_INDEX);
        recordStore.saveRecord(createMultiEntryMapDoc(1623L, ENGINEER_JOKE, "sampleTextPhrase", WAYLON, "sampleTextSong", 2));
        RecordCursor<IndexEntry> indexEntries = recordStore.scanIndex(MAP_ON_VALUE_INDEX, IndexScanType.BY_LUCENE, TupleRange.allOf(Tuple.from("entry_value:Vision", "sampleTextPhrase")), null, ScanProperties.FORWARD_SCAN);
        assertEquals(1, indexEntries.getCount().join());
        assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LOAD_SCAN_ENTRY).getCount());
        assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(MAP_ON_VALUE_INDEX).subspace(Tuple.from("sampleTextPhrase")), context, "_0.cfs", true);
    }
}
Also used : FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) IndexEntry(com.apple.foundationdb.record.IndexEntry) Test(org.junit.jupiter.api.Test)

Example 59 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method simpleInsertAndSearch.

@Test
void simpleInsertAndSearch() {
    try (FDBRecordContext context = openContext()) {
        rebuildIndexMetaData(context, SIMPLE_DOC, SIMPLE_TEXT_SUFFIXES);
        recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
        recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
        RecordCursor<IndexEntry> indexEntries = recordStore.scanIndex(SIMPLE_TEXT_SUFFIXES, IndexScanType.BY_LUCENE_FULL_TEXT, TupleRange.allOf(Tuple.from("Vision")), null, ScanProperties.FORWARD_SCAN);
        assertEquals(1, recordStore.scanIndex(SIMPLE_TEXT_SUFFIXES, IndexScanType.BY_LUCENE_FULL_TEXT, TupleRange.allOf(Tuple.from("Vision")), null, ScanProperties.FORWARD_SCAN).getCount().join());
        assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LOAD_SCAN_ENTRY).getCount());
        assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(SIMPLE_TEXT_SUFFIXES), context, "_0.cfs", true);
    }
}
Also used : FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) IndexEntry(com.apple.foundationdb.record.IndexEntry) Test(org.junit.jupiter.api.Test)

Example 60 with IndexEntry

use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.

the class LuceneIndexTest method searchForAutoCompleteWithTextSizeLimit.

void searchForAutoCompleteWithTextSizeLimit(int limit, boolean matches) throws Exception {
    try (FDBRecordContext context = openContext(RecordLayerPropertyStorage.newBuilder().addProp(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT, limit))) {
        final RecordType recordType = addIndexAndSaveRecordForAutoComplete(context, false);
        List<IndexEntry> results = recordStore.scanIndex(SIMPLE_TEXT_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from("software engineer")), null, ScanProperties.FORWARD_SCAN).asList().get();
        if (!matches) {
            // Assert no suggestions
            assertTrue(results.isEmpty());
            return;
        }
        // Assert the count of suggestions
        assertEquals(1, results.size());
        // Assert the suggestions' keys
        List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
        assertEquals(ImmutableList.of(ENGINEER_JOKE), suggestions);
        // Assert the corresponding field for the suggestions
        List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
        assertEquals(ImmutableList.of("text"), fields);
        results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
        assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
        assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(SIMPLE_TEXT_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
    }
}
Also used : IndexEntry(com.apple.foundationdb.record.IndexEntry) Arrays(java.util.Arrays) StoreTimer(com.apple.foundationdb.record.provider.common.StoreTimer) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) Random(java.util.Random) SynonymAnalyzer(com.apple.foundationdb.record.lucene.synonym.SynonymAnalyzer) EnglishSynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.EnglishSynonymMapConfig) Subspace(com.apple.foundationdb.subspace.Subspace) IndexScanType(com.apple.foundationdb.record.IndexScanType) Tuple(com.apple.foundationdb.tuple.Tuple) Pair(org.apache.commons.lang3.tuple.Pair) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) Expressions.concatenateFields(com.apple.foundationdb.record.metadata.Key.Expressions.concatenateFields) COMPLEX_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC) Expressions.concat(com.apple.foundationdb.record.metadata.Key.Expressions.concat) GroupingKeyExpression(com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression) Tag(org.junit.jupiter.api.Tag) SynonymMapConfig(com.apple.foundationdb.record.lucene.synonym.SynonymMapConfig) KeyExpression(com.apple.foundationdb.record.metadata.expressions.KeyExpression) IndexOptions(com.apple.foundationdb.record.metadata.IndexOptions) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Collectors(java.util.stream.Collectors) FDBRecordContextConfig(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContextConfig) IsCollectionWithSize.hasSize(org.hamcrest.collection.IsCollectionWithSize.hasSize) TupleRange(com.apple.foundationdb.record.TupleRange) Test(org.junit.jupiter.api.Test) TestRecordsTextProto(com.apple.foundationdb.record.TestRecordsTextProto) List(java.util.List) NgramAnalyzer(com.apple.foundationdb.record.lucene.ngram.NgramAnalyzer) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) RecordCursorProto(com.apple.foundationdb.record.RecordCursorProto) FDBDirectory(com.apple.foundationdb.record.lucene.directory.FDBDirectory) IndexTypes(com.apple.foundationdb.record.metadata.IndexTypes) Expressions.function(com.apple.foundationdb.record.metadata.Key.Expressions.function) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) RecordLayerPropertyStorage(com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage) Descriptors(com.google.protobuf.Descriptors) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) ExecuteProperties(com.apple.foundationdb.record.ExecuteProperties) FDBRecordStore(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore) IndexKeyValueToPartialRecord(com.apple.foundationdb.record.query.plan.IndexKeyValueToPartialRecord) ImmutableList(com.google.common.collect.ImmutableList) ScanProperties(com.apple.foundationdb.record.ScanProperties) FDBLuceneFileReference(com.apple.foundationdb.record.lucene.directory.FDBLuceneFileReference) TextIndexTestUtils(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Nonnull(javax.annotation.Nonnull) Expressions.field(com.apple.foundationdb.record.metadata.Key.Expressions.field) Nullable(javax.annotation.Nullable) SIMPLE_DOC(com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC) FDBStoredRecord(com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord) RecordMetaDataBuilder(com.apple.foundationdb.record.RecordMetaDataBuilder) Tags(com.apple.test.Tags) SequenceInputStream(java.io.SequenceInputStream) IndexFileNames(org.apache.lucene.index.IndexFileNames) IOException(java.io.IOException) FDBRecordStoreTestBase(com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase) ExecutionException(java.util.concurrent.ExecutionException) RecordType(com.apple.foundationdb.record.metadata.RecordType) Index(com.apple.foundationdb.record.metadata.Index) AutoService(com.google.auto.service.AutoService) Message(com.google.protobuf.Message) Assertions(org.junit.jupiter.api.Assertions) RecordCursor(com.apple.foundationdb.record.RecordCursor) Collections(java.util.Collections) AllSuffixesTextTokenizer(com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer) InputStream(java.io.InputStream) RecordType(com.apple.foundationdb.record.metadata.RecordType) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) IndexEntry(com.apple.foundationdb.record.IndexEntry)

Aggregations

IndexEntry (com.apple.foundationdb.record.IndexEntry)74 Tuple (com.apple.foundationdb.tuple.Tuple)41 Nonnull (javax.annotation.Nonnull)37 Index (com.apple.foundationdb.record.metadata.Index)34 Test (org.junit.jupiter.api.Test)34 FDBRecordContext (com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext)32 Message (com.google.protobuf.Message)32 ScanProperties (com.apple.foundationdb.record.ScanProperties)29 ArrayList (java.util.ArrayList)29 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)28 TupleRange (com.apple.foundationdb.record.TupleRange)28 List (java.util.List)28 Nullable (javax.annotation.Nullable)27 IndexScanType (com.apple.foundationdb.record.IndexScanType)24 RecordCursor (com.apple.foundationdb.record.RecordCursor)22 CompletableFuture (java.util.concurrent.CompletableFuture)21 Collectors (java.util.stream.Collectors)21 ExecuteProperties (com.apple.foundationdb.record.ExecuteProperties)20 TupleHelpers (com.apple.foundationdb.tuple.TupleHelpers)20 FDBStoreTimer (com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer)19