use of com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method queryComplexDocumentsWithAdditionalFilters.
@Test
public void queryComplexDocumentsWithAdditionalFilters() throws Exception {
final List<String> textSamples = Arrays.asList(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.GERMAN, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.YIDDISH, "Napoleon and the Duke of Wellington met in Waterloo in 1815.");
final List<ComplexDocument> documents = IntStream.range(0, textSamples.size()).mapToObj(i -> ComplexDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).addTag("3:" + (i % 3)).setScore(i).build()).collect(Collectors.toList());
final Index rankIndex = new Index("Complex$rank(score)", field("score").groupBy(field("group")), IndexTypes.RANK);
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
final RecordTypeBuilder complexDocRecordType = metaDataBuilder.getRecordType(COMPLEX_DOC);
metaDataBuilder.addIndex(complexDocRecordType, COMPLEX_TEXT_BY_GROUP);
metaDataBuilder.addIndex(complexDocRecordType, rankIndex);
});
documents.forEach(recordStore::saveRecord);
assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.field("tag").oneOfThem().equalsValue("3:2"), 1, 758136568));
assertEquals(Arrays.asList(Tuple.from(1L, 1L), Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.field("text").text().containsPrefix("continu"), 1, -1043653062));
assertEquals(Collections.singletonList(Tuple.from(1L, 7L)), queryComplexDocumentsWithIndex(Query.field("text").text().contains("napoleon"), Query.and(Query.field("text").text().containsPrefix("th"), Query.field("text").text().contains("waterloo")), 1, -754900112));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.not(Query.field("tag").oneOfThem().equalsValue("3:2")), 1, 758136569));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.not(Query.field("tag").oneOfThem().equalsValue("3:2")), 1, 758136569));
assertEquals(Arrays.asList(Tuple.from(0L, 0L), Tuple.from(0L, 6L)), queryComplexDocumentsWithOr((OrComponent) Query.or(Query.field("text").text().containsAll("unit named after"), Query.field("text").text().containsPhrase("אן ארמיי און פלאט")), 0, -1558384887));
assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.or(Query.field("tag").oneOfThem().equalsValue("3:2"), Query.field("tag").oneOfThem().equalsValue("3:0")), true, 1, -27568755));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("fearful passage love", 7), Query.rank(field("score").groupBy(field("group"))).lessThan(2L), true, 1, -2132208833));
assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love", true), Query.field("tag").oneOfThem().equalsValue("3:2"), true, 1, -419325379));
assertEquals(Collections.singletonList(Tuple.from(1L, 5L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love", false), Query.field("tag").oneOfThem().equalsValue("3:2"), false, 1, -1902024530));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("fear pass love"), Query.rank(field("score").groupBy(field("group"))).lessThan(2L), true, 1, 669157421));
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveComplexMultiDocuments.
@Test
public void saveComplexMultiDocuments() throws Exception {
final ComplexDocument shakespeareDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).addTag("a").addTag("b").build();
// First, repeated key comes *before* text.
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_MULTI_TAG_INDEX));
recordStore.saveRecord(shakespeareDocument);
int indexKeys = getSaveIndexKeyCount(recordStore);
assertEquals(164, indexKeys);
List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_MULTI_TAG_INDEX, Tuple.from("a", "civil"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(0L, 1623L), Arrays.asList(22, 25))), entryList);
commit(context);
}
try (FDBRecordContext context = openContext()) {
FDBRecordStore.deleteStore(context, recordStore.getSubspace());
commit(context);
}
// Then, repeated key comes *after* text.
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_THEN_TAG_INDEX));
recordStore.saveRecord(shakespeareDocument);
int indexKeys = getSaveIndexKeyCount(recordStore);
assertEquals(164, indexKeys);
List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_THEN_TAG_INDEX, Tuple.from("civil"));
assertEquals(Arrays.asList(entryOf(Tuple.from("a", 0L, 1623L), Arrays.asList(22, 25)), entryOf(Tuple.from("b", 0L, 1623L), Arrays.asList(22, 25))), entryList);
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveComplexWithAggressiveConflictRanges.
@Test
public void saveComplexWithAggressiveConflictRanges() throws Exception {
// These two documents are in different groups, so even with aggressive conflict
// ranges, they should be able to be committed concurrently.
final ComplexDocument zeroGroupDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).build();
final ComplexDocument oneGroupDocument = ComplexDocument.newBuilder().setGroup(1).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).build();
final RecordMetaDataHook hook = metaDataBuilder -> {
final Index newIndex = new Index(COMPLEX_TEXT_BY_GROUP.getName(), COMBINED_TEXT_BY_GROUP.getRootExpression(), IndexTypes.TEXT, ImmutableMap.of(IndexOptions.TEXT_ADD_AGGRESSIVE_CONFLICT_RANGES_OPTION, "true"));
metaDataBuilder.addIndex(COMPLEX_DOC, newIndex);
};
saveTwoRecordsConcurrently(hook, zeroGroupDocument, oneGroupDocument, true);
}
use of com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveComplexDocuments.
@Test
public void saveComplexDocuments() throws Exception {
ComplexDocument complexDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1066L).setText("Very complex. Not to be trifled with.").build();
ComplexDocument shakespeareDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).addTag("a").addTag("b").build();
ComplexDocument yiddishDocument = ComplexDocument.newBuilder().setGroup(1).setDocId(1944L).setText(TextSamples.YIDDISH).addTag("c").build();
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_TEXT_BY_GROUP));
recordStore.saveRecord(complexDocument);
int firstKeys = getSaveIndexKeyCount(recordStore);
assertEquals(complexDocument.getText().split(" ").length, firstKeys);
recordStore.saveRecord(shakespeareDocument);
int secondKeys = getSaveIndexKeyCount(recordStore) - firstKeys;
assertEquals(82, secondKeys);
recordStore.saveRecord(yiddishDocument);
int thirdKeys = getSaveIndexKeyCount(recordStore) - secondKeys - firstKeys;
assertEquals(9, thirdKeys);
List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_TEXT_BY_GROUP, Tuple.from(0L, "to"));
assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(3)), entryOf(Tuple.from(1623L), Arrays.asList(18, 108))), entryList);
List<Message> recordList = recordStore.scanIndexRecords(COMPLEX_TEXT_BY_GROUP.getName(), BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(0L, "to")), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
assertEquals(Arrays.asList(complexDocument, shakespeareDocument), recordList);
entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.prefixedBy("א").prepend(Tuple.from(1L))), null);
assertEquals(Arrays.asList(entryOf(Tuple.from(1L, "א", 1944L), Arrays.asList(0, 3)), entryOf(Tuple.from(1L, "און", 1944L), Collections.singletonList(8)), entryOf(Tuple.from(1L, "איז", 1944L), Collections.singletonList(2)), entryOf(Tuple.from(1L, "אן", 1944L), Collections.singletonList(6)), entryOf(Tuple.from(1L, "ארמיי", 1944L), Collections.singletonList(7))), entryList);
// Read the whole store and make sure the values come back in a somewhat sensible way
entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.ALL), null);
assertEquals(firstKeys + secondKeys + thirdKeys, entryList.size());
int i = 0;
String last = null;
for (Map.Entry<Tuple, List<Integer>> entry : entryList) {
assertEquals(3, entry.getKey().size());
if (i < firstKeys + secondKeys) {
assertEquals(0L, entry.getKey().getLong(0));
assertThat(entry.getKey().getLong(2), anyOf(is(1066L), is(1623L)));
} else {
assertEquals(1L, entry.getKey().getLong(0));
assertEquals(1944L, entry.getKey().getLong(2));
if (i == firstKeys + secondKeys) {
last = null;
}
}
if (last != null) {
assertThat(entry.getKey().getString(1), greaterThanOrEqualTo(last));
}
last = entry.getKey().getString(1);
i++;
}
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.ComplexDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method queryComplexDocuments.
@Test
public void queryComplexDocuments() throws Exception {
final List<String> textSamples = Arrays.asList(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH);
final List<ComplexDocument> documents = IntStream.range(0, textSamples.size()).mapToObj(i -> ComplexDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build()).collect(Collectors.toList());
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
final RecordTypeBuilder complexDocRecordType = metaDataBuilder.getRecordType(COMPLEX_DOC);
metaDataBuilder.addIndex(complexDocRecordType, COMPLEX_TEXT_BY_GROUP);
});
documents.forEach(recordStore::saveRecord);
assertEquals(Collections.singletonList(Tuple.from(0L, 0L)), queryComplexDocumentsWithIndex(Query.field("text").text().contains("angstrom"), 0, 372972877));
assertEquals(Collections.emptyList(), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("civil blood parents' strife"), 0L, -1615886689));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("civil blood parents' strife"), 1L, -1615886658));
assertEquals(Collections.emptyList(), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("civil blood parents' strife", 4), 1L, -1436111364));
assertEquals(Collections.singletonList(Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAll("civil blood parents' strife", 35), 1L, -1436081573));
assertEquals(Arrays.asList(Tuple.from(0L, 0L), Tuple.from(0L, 2L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAny("angstrom parents king napoleons"), 0L, -1092421072));
assertEquals(Collections.singletonList(Tuple.from(1L, 3L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsPhrase("recu un Thiers"), 1L, 1395848801));
assertEquals(Collections.singletonList(Tuple.from(0L, 0L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsPrefix("ang"), 0L, -1013515738));
assertEquals(Arrays.asList(Tuple.from(1L, 3L), Tuple.from(1L, 1L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsPrefix("un"), 1L, -995158140));
assertEquals(ImmutableSet.of(Tuple.from(0L, 0L), Tuple.from(0L, 2L)), ImmutableSet.copyOf(queryComplexDocumentsWithIndex(Query.field("text").text().containsAnyPrefix("ang par nap kin"), 0L, -1089713854)));
assertEquals(Collections.singletonList(Tuple.from(0L, 0L)), queryComplexDocumentsWithIndex(Query.field("text").text().containsAllPrefixes("ang uni name", false), 0L, 646414402));
commit(context);
}
}
Aggregations