use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method searchForSpellcheckForGroupedRecord.
@Test
void searchForSpellcheckForGroupedRecord() throws Exception {
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX);
});
FDBStoredRecord<Message> fdbRecord = recordStore.saveRecord(createMultiEntryMapDoc(1623L, ENGINEER_JOKE, "sampleTextPhrase", WAYLON, "sampleTextSong", 2));
List<IndexEntry> indexEntries = recordStore.scanIndex(MAP_ON_VALUE_INDEX, IndexScanType.BY_LUCENE_SPELLCHECK, TupleRange.allOf(Tuple.from("Visin", "sampleTextPhrase")), null, ScanProperties.FORWARD_SCAN).asList().get();
assertEquals(1, indexEntries.size());
IndexEntry indexEntry = indexEntries.get(0);
assertEquals(0.8F, indexEntry.getValue().get(0));
Descriptors.Descriptor recordDescriptor = TestRecordsTextProto.MapDocument.getDescriptor();
IndexKeyValueToPartialRecord toPartialRecord = LuceneIndexQueryPlan.getToPartialRecord(MAP_ON_VALUE_INDEX, fdbRecord.getRecordType(), IndexScanType.BY_LUCENE_SPELLCHECK);
Message message = toPartialRecord.toRecord(recordDescriptor, indexEntry);
Descriptors.FieldDescriptor entryDescriptor = recordDescriptor.findFieldByName("entry");
Message entry = (Message) message.getRepeatedField(entryDescriptor, 0);
Descriptors.FieldDescriptor keyDescriptor = entryDescriptor.getMessageType().findFieldByName("key");
Descriptors.FieldDescriptor valueDescriptor = entryDescriptor.getMessageType().findFieldByName("value");
// TODO: This seems like the wrong field string to return. I'm not sure what to do here
assertEquals("sampleTextPhrase", entry.getField(keyDescriptor));
assertEquals("vision", entry.getField(valueDescriptor));
// assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
// assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(MAP_ON_VALUE_INDEX_WITH_AUTO_COMPLETE).subspace(Tuple.from("sampleTextPhrase")), context, "_0.cfs", true);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method searchForAutoCompleteAndAssert.
private void searchForAutoCompleteAndAssert(String query, boolean matches, boolean highlight, int textSizeLimit) throws Exception {
try (FDBRecordContext context = openContext(RecordLayerPropertyStorage.newBuilder().addProp(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT, textSizeLimit))) {
final RecordType recordType = addIndexAndSaveRecordForAutoComplete(context, highlight);
List<IndexEntry> results = recordStore.scanIndex(highlight ? SIMPLE_TEXT_WITH_AUTO_COMPLETE_WITH_HIGHLIGHT : SIMPLE_TEXT_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from(query)), null, ScanProperties.FORWARD_SCAN).asList().get();
if (!matches) {
// Assert no suggestions
assertTrue(results.isEmpty());
assertEquals(0, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
return;
}
// Assert the count of suggestions
assertEquals(6, results.size());
// Assert the suggestions' keys
List<String> suggestions = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 1)).collect(Collectors.toList());
if (highlight) {
assertEquals(ImmutableList.of("<b>good</b> evening", "<b>Good</b> night", "<b>Good</b> morning", "<b>Good</b> afternoon", "I'm <b>good</b>", "That's really <b>good</b>!"), suggestions);
} else {
assertEquals(ImmutableList.of("good evening", "Good night", "Good morning", "Good afternoon", "I'm good", "That's really good!"), suggestions);
}
// Assert the corresponding field for the suggestions
List<String> fields = results.stream().map(i -> (String) i.getKey().get(i.getKeySize() - 2)).collect(Collectors.toList());
assertEquals(ImmutableList.of("text", "text", "text", "text", "text", "text"), fields);
// Assert the suggestions are sorted according to their values, which are determined by the position of the term into the indexed text
List<Long> values = results.stream().map(i -> (Long) i.getValue().get(0)).collect(Collectors.toList());
List<Long> valuesSorted = new ArrayList<>(values);
Collections.sort(valuesSorted, Collections.reverseOrder());
assertEquals(valuesSorted, values);
assertEquals(values.get(0), values.get(1));
assertEquals(values.get(1), values.get(2));
assertEquals(values.get(2), values.get(3));
assertTrue(values.get(3) > values.get(4));
assertTrue(values.get(4) > values.get(5));
results.stream().forEach(i -> assertDocumentPartialRecordFromIndexEntry(recordType, i, (String) i.getKey().get(i.getKeySize() - 1), (String) i.getKey().get(i.getKeySize() - 2), IndexScanType.BY_LUCENE_AUTO_COMPLETE));
assertEquals(6, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
assertEntriesAndSegmentInfoStoredInCompoundFile(AutoCompleteSuggesterCommitCheckAsync.getSuggestionIndexSubspace(recordStore.indexSubspace(SIMPLE_TEXT_WITH_AUTO_COMPLETE), TupleHelpers.EMPTY), context, "_0.cfs", true);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method testNestedFieldSearch.
@Test
void testNestedFieldSearch() {
try (FDBRecordContext context = openContext()) {
rebuildIndexMetaData(context, MAP_DOC, MAP_ON_VALUE_INDEX);
recordStore.saveRecord(createComplexMapDocument(1623L, ENGINEER_JOKE, "sampleTextSong", 2));
recordStore.saveRecord(createComplexMapDocument(1547L, WAYLON, "sampleTextPhrase", 1));
RecordCursor<IndexEntry> indexEntries = recordStore.scanIndex(MAP_ON_VALUE_INDEX, IndexScanType.BY_LUCENE, TupleRange.allOf(Tuple.from("entry_value:Vision", "sampleTextSong")), null, ScanProperties.FORWARD_SCAN);
assertEquals(1, indexEntries.getCount().join());
assertEquals(1, context.getTimer().getCounter(FDBStoreTimer.Counts.LOAD_SCAN_ENTRY).getCount());
assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(MAP_ON_VALUE_INDEX).subspace(Tuple.from("sampleTextSong")), context, "_0.cfs", true);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method testContinuation.
@Test
void testContinuation() {
try (FDBRecordContext context = openContext()) {
rebuildIndexMetaData(context, SIMPLE_DOC, SIMPLE_TEXT_SUFFIXES);
recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
recordStore.saveRecord(createSimpleDocument(1624L, ENGINEER_JOKE, 2));
recordStore.saveRecord(createSimpleDocument(1625L, ENGINEER_JOKE, 2));
recordStore.saveRecord(createSimpleDocument(1626L, ENGINEER_JOKE, 2));
recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
RecordCursorProto.LuceneIndexContinuation continuation = RecordCursorProto.LuceneIndexContinuation.newBuilder().setDoc(1).setScore(0.27130663F).setShard(0).build();
RecordCursor<IndexEntry> recordCursor = recordStore.scanIndex(SIMPLE_TEXT_SUFFIXES, IndexScanType.BY_LUCENE_FULL_TEXT, TupleRange.allOf(Tuple.from("Vision")), continuation.toByteArray(), ScanProperties.FORWARD_SCAN);
assertEquals(2, recordCursor.getCount().join());
assertEntriesAndSegmentInfoStoredInCompoundFile(recordStore.indexSubspace(SIMPLE_TEXT_SUFFIXES), context, "_0.cfs", true);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class LuceneIndexTest method searchForAutoCompleteWithLoadingNoRecords.
/**
* To verify the suggestion lookup can work correctly if the suggester is never built and no entries exist in the directory.
*/
@Test
void searchForAutoCompleteWithLoadingNoRecords() throws Exception {
try (FDBRecordContext context = openContext(RecordLayerPropertyStorage.newBuilder().addProp(LuceneRecordContextProperties.LUCENE_AUTO_COMPLETE_TEXT_SIZE_UPPER_LIMIT, DEFAULT_AUTO_COMPLETE_TEXT_SIZE_LIMIT))) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(SIMPLE_DOC, SIMPLE_TEXT_WITH_AUTO_COMPLETE);
});
List<IndexEntry> results = recordStore.scanIndex(SIMPLE_TEXT_WITH_AUTO_COMPLETE, IndexScanType.BY_LUCENE_AUTO_COMPLETE, TupleRange.allOf(Tuple.from("hello")), null, ScanProperties.FORWARD_SCAN).asList().get();
assertTrue(results.isEmpty());
assertEquals(0, context.getTimer().getCounter(FDBStoreTimer.Counts.LUCENE_SCAN_MATCHED_AUTO_COMPLETE_SUGGESTIONS).getCount());
}
}
Aggregations