use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method queryMapsWithGroups.
@Test
public void queryMapsWithGroups() throws Exception {
final List<String> textSamples = Arrays.asList(TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.ANGSTROM);
final List<MapDocument> documents = IntStream.range(0, textSamples.size() / 2).mapToObj(i -> MapDocument.newBuilder().setDocId(i).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(textSamples.get(i * 2)).build()).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(textSamples.get(i * 2 + 1)).build()).setGroup(i % 2).build()).collect(Collectors.toList());
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_GROUPED_INDEX));
documents.forEach(recordStore::saveRecord);
assertEquals(Collections.singletonList(1L), queryMapDocumentsWithGroupedIndex("a", Query.field("value").text().containsPhrase("both alike in dignity"), 1L, 1376087127));
assertEquals(Collections.singletonList(0L), queryMapDocumentsWithGroupedIndex("b", Query.field("value").text().containsAny("king anders"), 0L, -1204479544));
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method queryMapDocuments.
@Test
public void queryMapDocuments() throws Exception {
final List<String> textSamples = Arrays.asList(TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.ANGSTROM, TextSamples.AETHELRED, TextSamples.FRENCH);
final List<MapDocument> documents = IntStream.range(0, textSamples.size() / 2).mapToObj(i -> MapDocument.newBuilder().setDocId(i).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(textSamples.get(i * 2)).build()).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(textSamples.get(i * 2 + 1)).build()).setGroup(i % 2).build()).collect(Collectors.toList());
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX));
documents.forEach(recordStore::saveRecord);
assertEquals(Collections.singletonList(2L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAny("king unknown_token"), 1059912699, true));
assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034960, true));
assertEquals(Collections.emptyList(), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPhrase("civil blood makes civil hands unclean"), 1085034991, true));
assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsPrefix("na"), 1125182095, true));
assertEquals(Arrays.asList(0L, 1L), queryMapDocumentsWithIndex("a", Query.field("value").text().containsAllPrefixes("civ mut ha"), 0, false));
assertEquals(Arrays.asList(1L, 2L), queryMapDocumentsWithIndex("b", Query.field("value").text().containsAnyPrefix("civ mut na"), 0, true));
RecordQuery queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.and(Query.field("group").equalsValue(0L), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsAny("anders king"))))).build();
RecordQueryPlan planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
assertThat(planWithAdditionalFilter, filter(Query.field("group").equalsValue(0L), descendant(textIndexScan(anything()))));
List<Long> queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
assertEquals(Collections.singletonList(0L), queryResults);
queryWithAdditionalFilter = RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(Query.or(Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("a"), Query.field("value").text().containsPhrase("bury their parents strife"))), Query.field("entry").oneOfThem().matches(Query.and(Query.field("key").equalsValue("b"), Query.field("value").text().containsPrefix("th"))))).build();
planWithAdditionalFilter = recordStore.planQuery(queryWithAdditionalFilter);
assertThat(planWithAdditionalFilter, primaryKeyDistinct(unorderedUnion(descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))), descendant(textIndexScan(indexName(equalTo(MAP_ON_VALUE_INDEX.getName())))))));
queryResults = recordStore.executeQuery(planWithAdditionalFilter).map(FDBQueriedRecord::getPrimaryKey).map(tuple -> tuple.getLong(0)).asList().join();
assertEquals(3, queryResults.size());
assertEquals(ImmutableSet.of(0L, 1L, 2L), ImmutableSet.copyOf(queryResults));
// Planner bug that can happen with certain malformed queries. This plan actually
// returns records where the key and the value match in the same entry, but it is
// asking for all records where *any* entry has a key matching "a" and *any* entry
// has a value matching the text predicate. In reality, this is probably a sign
// the user didn't input their query correctly, but it requires more work from the
// planner not to plan this kind of query.
// FIXME: Full Text: The Planner doesn't always correctly handle ands with nesteds (https://github.com/FoundationDB/fdb-record-layer/issues/53)
final QueryComponent malformedMapFilter = Query.and(Query.field("entry").oneOfThem().matches(Query.field("key").equalsValue("a")), Query.field("entry").oneOfThem().matches(Query.field("value").text().containsAll("civil hands unclean")));
RecordQueryPlan malformedMapPlan = planner.plan(RecordQuery.newBuilder().setRecordType(MAP_DOC).setFilter(malformedMapFilter).build());
assertThat(malformedMapPlan, descendant(textIndexScan(allOf(indexName(MAP_ON_VALUE_INDEX.getName()), groupingBounds(allOf(notNullValue(), hasTupleString("[[a],[a]]"))), textComparison(equalTo(new Comparisons.TextComparison(Comparisons.Type.TEXT_CONTAINS_ALL, "civil hands unclean", null, DefaultTextTokenizer.NAME)))))));
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveMapDocuments.
@Test
public void saveMapDocuments() throws Exception {
final MapDocument firstDocument = MapDocument.newBuilder().setDocId(1066L).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(TextSamples.ANGSTROM)).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue(TextSamples.ROMEO_AND_JULIET_PROLOGUE)).build();
final MapDocument secondDocument = MapDocument.newBuilder().setDocId(1415L).addEntry(MapDocument.Entry.newBuilder().setKey("a").setValue(TextSamples.OLD_S)).addEntry(MapDocument.Entry.newBuilder().setKey("c").setValue(TextSamples.FRENCH)).build();
final MapDocument documentWithEmpties = MapDocument.newBuilder().setDocId(1815L).addEntry(MapDocument.Entry.newBuilder().setKey(// value intentionally left unset
"a")).addEntry(MapDocument.Entry.newBuilder().setKey("b").setValue("")).build();
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_INDEX);
});
recordStore.saveRecord(firstDocument);
recordStore.saveRecord(secondDocument);
recordStore.saveRecord(documentWithEmpties);
assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(0)), entryOf(Tuple.from(1415L), Collections.singletonList(7))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("a", "the")));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Arrays.asList(22, 25))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("b", "civil")));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), Collections.singletonList(4))), scanMapEntries(recordStore, MAP_ON_VALUE_INDEX, Tuple.from("c", "a")));
commit(context);
}
}
use of com.apple.foundationdb.record.TestRecordsTextProto.MapDocument in project fdb-record-layer by FoundationDB.
the class TextIndexTest method tokenizerVersionChangeWithMultipleEntries.
@Test
public void tokenizerVersionChangeWithMultipleEntries() throws Exception {
final MapDocument map1 = MapDocument.newBuilder().setDocId(1066L).addEntry(MapDocument.Entry.newBuilder().setKey("fr").setValue(TextSamples.FRENCH)).addEntry(MapDocument.Entry.newBuilder().setKey("de").setValue(TextSamples.GERMAN)).build();
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_PREFIX_LEGACY);
});
recordStore.saveRecord(map1);
List<Map.Entry<Tuple, List<Integer>>> scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX_LEGACY, Tuple.from("fr", "rec"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(5))), scannedEntries);
scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX_LEGACY, Tuple.from("de", "wah"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(8))), scannedEntries);
assertEquals(Collections.singletonList(Pair.of(Tuple.from(1066L), 0)), scanTokenizerVersions(recordStore, MAP_ON_VALUE_PREFIX_LEGACY));
commit(context);
}
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> {
metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
metaDataBuilder.addIndex(MAP_DOC, MAP_ON_VALUE_PREFIX);
});
// save an document where most of the source entries remain the same but are re-indexed because of the tokenizer version change
final MapDocument map1prime = map1.toBuilder().addEntry(MapDocument.Entry.newBuilder().setKey("yi").setValue(TextSamples.YIDDISH)).build();
recordStore.saveRecord(map1prime);
List<Map.Entry<Tuple, List<Integer>>> scannedEntries = scanMapEntries(recordStore, MAP_ON_VALUE_PREFIX, Tuple.from("yi", "שפרא"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(1))), scannedEntries);
scannedEntries = toMapEntries(scanIndex(recordStore, MAP_ON_VALUE_PREFIX, TupleRange.prefixedBy("rec").prepend(Tuple.from("fr"))), Tuple.from("fr"));
assertEquals(Collections.singletonList(entryOf(Tuple.from("recu", 1066L), Collections.singletonList(5))), scannedEntries);
scannedEntries = toMapEntries(scanIndex(recordStore, MAP_ON_VALUE_PREFIX, TupleRange.prefixedBy("wah").prepend(Tuple.from("de"))), Tuple.from("de"));
assertEquals(Collections.singletonList(entryOf(Tuple.from("wahr", 1066L), Collections.singletonList(8))), scannedEntries);
assertEquals(Collections.singletonList(Pair.of(Tuple.from(1066L), 1)), scanTokenizerVersions(recordStore, MAP_ON_VALUE_PREFIX_LEGACY));
commit(context);
}
}
Aggregations