use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.
the class FDBCrossRecordQueryTest method testMultiRecordTypeIndexScan.
/**
* Verify that multi-type record queries can scan multi-type indexes.
* Verify that single-type record queries can scan multi-type indexes with a type filter.
*/
@Test
public void testMultiRecordTypeIndexScan() throws Exception {
try (FDBRecordContext context = openContext()) {
openUnionRecordStore(context);
saveSimpleRecord(100, "first", 1);
saveSimpleRecord(110, "second", 2);
saveSimpleRecord2("third", 3);
saveSimpleRecord2("fourth", 4);
saveSimpleRecord(80, "fifth", 5);
saveSimpleRecord2("sixth", 6);
saveSimpleRecord2("seventh", 7);
saveSimpleRecord(60, "seventh", 7);
saveSimpleRecord2("seventh again", 7);
saveSimpleRecord3("t3 second", 2);
saveSimpleRecord3("t3 sixth", 6);
saveSimpleRecord3("t3 seventh", 7);
commit(context);
}
List<String> names = new ArrayList<>();
List<Integer> etags = new ArrayList<>();
try (FDBRecordContext context = openContext()) {
openUnionRecordStore(context);
try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("partial_versions", IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asIterator()) {
while (cursor.hasNext()) {
final Message record = cursor.next().getRecord();
names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
}
}
assertDiscardedNone(context);
}
assertEquals(Arrays.asList("first", "second", "third", "fourth", "fifth", "sixth"), names.subList(0, 6));
assertThat(names.subList(6, 9), containsInAnyOrder("seventh", "seventh", "seventh again"));
assertEquals(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 7, 7), etags);
{
RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord", "MySimpleRecord2")).setFilter(Query.field("etag").equalsValue(7)).build();
// Index(partial_versions [[7],[7]])
RecordQueryPlan plan = planner.plan(query);
MatcherAssert.assertThat(plan, indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]")))));
assertEquals(-501898489, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(-1416119651, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-1071937908, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
names.clear();
etags.clear();
try (FDBRecordContext context = openContext()) {
openUnionRecordStore(context);
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
while (cursor.hasNext()) {
final Message record = cursor.next().getRecord();
names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
}
}
assertDiscardedNone(context);
}
assertThat(names, containsInAnyOrder("seventh", "seventh", "seventh again"));
assertEquals(Arrays.asList(7, 7, 7), etags);
}
{
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord2").setFilter(Query.field("etag").equalsValue(7)).build();
// Index(partial_versions [[7],[7]]) | [MySimpleRecord2]
RecordQueryPlan plan = planner.plan(query);
MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord2"), indexScan(allOf(indexName("partial_versions"), bounds(hasTupleString("[[7],[7]]"))))));
assertEquals(-1724404567, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(1091241424, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-1124026431, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
names.clear();
etags.clear();
try (FDBRecordContext context = openContext()) {
openUnionRecordStore(context);
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
while (cursor.hasNext()) {
final Message record = cursor.next().getRecord();
names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
}
}
assertDiscardedAtMost(1, context);
}
assertThat(names, containsInAnyOrder("seventh", "seventh again"));
assertEquals(Arrays.asList(7, 7), etags);
}
{
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord3").setFilter(Query.field("etag").equalsValue(7)).build();
// Index(versions [[7],[7]]) | [MySimpleRecord3]
RecordQueryPlan plan = planner.plan(query);
MatcherAssert.assertThat(plan, typeFilter(contains("MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
assertEquals(-1908726868, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(168009743, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-2047258112, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
names.clear();
etags.clear();
try (FDBRecordContext context = openContext()) {
clearStoreCounter(context);
openUnionRecordStore(context);
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
while (cursor.hasNext()) {
final Message record = cursor.next().getRecord();
names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
}
}
assertDiscardedAtMost(3, context);
}
assertEquals(Arrays.asList("t3 seventh"), names);
assertEquals(Arrays.asList(7), etags);
}
{
RecordQuery query = RecordQuery.newBuilder().setRecordTypes(Arrays.asList("MySimpleRecord2", "MySimpleRecord3")).setFilter(Query.field("etag").equalsValue(7)).build();
// Index(versions [[7],[7]]) | [MySimpleRecord2, MySimpleRecord3]
RecordQueryPlan plan = planner.plan(query);
MatcherAssert.assertThat(plan, typeFilter(containsInAnyOrder("MySimpleRecord2", "MySimpleRecord3"), indexScan(allOf(indexName("versions"), bounds(hasTupleString("[[7],[7]]"))))));
assertEquals(-1151709653, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(925026958, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-1290240897, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
names.clear();
etags.clear();
try (FDBRecordContext context = openContext()) {
clearStoreCounter(context);
openUnionRecordStore(context);
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan).asIterator()) {
while (cursor.hasNext()) {
final Message record = cursor.next().getRecord();
names.add((String) record.getField(record.getDescriptorForType().findFieldByName("str_value_indexed")));
etags.add((int) record.getField(record.getDescriptorForType().findFieldByName("etag")));
}
}
assertDiscardedAtMost(1, context);
}
assertThat(names, containsInAnyOrder("seventh", "seventh again", "t3 seventh"));
assertEquals(Arrays.asList(7, 7, 7), etags);
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveSimpleDocuments.
@Test
public void saveSimpleDocuments() throws Exception {
final SimpleDocument simpleDocument = SimpleDocument.newBuilder().setDocId(1066L).setText("This is a simple document. There isn't much going on here, if I'm honest.").setGroup(0).build();
final SimpleDocument buffaloDocument = SimpleDocument.newBuilder().setDocId(1415L).setText("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo Buffalo buffalo buffalo.").setGroup(1).build();
final SimpleDocument shakespeareDocument = SimpleDocument.newBuilder().setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).setGroup(2).build();
final SimpleDocument noTextDocument = SimpleDocument.newBuilder().setDocId(0L).setGroup(0).build();
final SimpleDocument emptyDocument = SimpleDocument.newBuilder().setDocId(1L).setGroup(1).setText("").build();
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
Index index = recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
recordStore.saveRecord(simpleDocument);
final int firstKeys = getSaveIndexKeyCount(recordStore);
assertEquals(simpleDocument.getText().split(" ").length, firstKeys);
final int firstKeyBytesWritten = getSaveIndexKeyBytes(recordStore);
final int firstValueBytesWritten = getSaveIndexValueBytes(recordStore);
List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1066L), Collections.singletonList(4))), entryList);
resetTimer(recordStore);
recordStore.saveRecord(buffaloDocument);
final int secondKeys = getSaveIndexKeyCount(recordStore);
assertEquals(1, secondKeys);
entryList = scanMapEntries(recordStore, index, Tuple.from("buffalo"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1415L), IntStream.range(0, 11).boxed().collect(Collectors.toList()))), entryList);
resetTimer(recordStore);
recordStore.saveRecord(shakespeareDocument);
final int thirdKeys = getSaveIndexKeyCount(recordStore);
assertEquals(82, thirdKeys);
final int thirdBytesWritten = getSaveIndexKeyBytes(recordStore) + getSaveIndexValueBytes(recordStore);
entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1623L), Arrays.asList(57, 72))), entryList);
entryList = toMapEntries(scanIndex(recordStore, index, TupleRange.prefixedBy("h")), null);
assertEquals(Arrays.asList(entryOf(Tuple.from("hands", 1623), Collections.singletonList(26)), entryOf(Tuple.from("here", 1066), Collections.singletonList(10)), entryOf(Tuple.from("here", 1623), Collections.singletonList(101)), entryOf(Tuple.from("honest", 1066), Collections.singletonList(13)), entryOf(Tuple.from("hours", 1623), Collections.singletonList(87)), entryOf(Tuple.from("households", 1623), Collections.singletonList(1))), entryList);
List<Message> recordList = recordStore.scanIndexRecords(index.getName(), BY_TEXT_TOKEN, TupleRange.prefixedBy("h"), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
assertEquals(Arrays.asList(shakespeareDocument, simpleDocument, shakespeareDocument, simpleDocument, shakespeareDocument, shakespeareDocument), recordList);
resetTimer(recordStore);
recordStore.saveRecord(noTextDocument);
assertEquals(0, getSaveIndexKeyCount(recordStore));
assertEquals(0, getLoadIndexKeyCount(recordStore));
resetTimer(recordStore);
recordStore.saveRecord(emptyDocument);
assertEquals(0, getSaveIndexKeyCount(recordStore));
assertEquals(0, getLoadIndexKeyCount(recordStore));
resetTimer(recordStore);
recordStore.deleteRecord(Tuple.from(1623L));
// all deleted but four overlaps with first record
assertEquals(thirdKeys - 4, getDeleteIndexKeyCount(recordStore));
// four keys of overlap overwritten
assertEquals(4, getSaveIndexKeyCount(recordStore));
assertThat(getDeleteIndexKeyBytes(recordStore) + getDeleteIndexValueBytes(recordStore), allOf(greaterThan(thirdKeys - 1), lessThan(thirdBytesWritten)));
entryList = scanMapEntries(recordStore, index, Tuple.from("parents"));
assertEquals(Collections.emptyList(), entryList);
resetTimer(recordStore);
recordStore.saveRecord(simpleDocument.toBuilder().setDocId(1707L).build());
assertEquals(firstKeys * 2, getLoadIndexKeyCount(recordStore));
assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
// should overwrite all the same keys
assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
final int seventhValueBytesWritten = getSaveIndexValueBytes(recordStore);
// contains same info as first value bytes + extra keys, but not key prefixes
assertThat(seventhValueBytesWritten, allOf(greaterThan(firstValueBytesWritten), lessThan(firstKeyBytesWritten + firstValueBytesWritten)));
entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(4)), entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
resetTimer(recordStore);
recordStore.deleteRecord(Tuple.from(1066L));
assertEquals(firstKeys, getLoadIndexKeyCount(recordStore));
// each of the original keys are deleted
assertEquals(firstKeys, getDeleteIndexKeyCount(recordStore));
assertEquals(firstKeyBytesWritten, getDeleteIndexKeyBytes(recordStore));
assertEquals(firstValueBytesWritten + seventhValueBytesWritten, getDeleteIndexValueBytes(recordStore));
// a new set of keys are all written
assertEquals(firstKeys, getSaveIndexKeyCount(recordStore));
// they should have the same size (though their contents are different)
assertEquals(firstKeyBytesWritten, getSaveIndexKeyBytes(recordStore));
assertEquals(firstValueBytesWritten, getSaveIndexValueBytes(recordStore));
entryList = scanMapEntries(recordStore, index, Tuple.from("document"));
assertEquals(Collections.singletonList(entryOf(Tuple.from(1707L), Collections.singletonList(4))), entryList);
commit(context);
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.
the class TextIndexTest method saveComplexDocuments.
@Test
public void saveComplexDocuments() throws Exception {
ComplexDocument complexDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1066L).setText("Very complex. Not to be trifled with.").build();
ComplexDocument shakespeareDocument = ComplexDocument.newBuilder().setGroup(0).setDocId(1623L).setText(TextSamples.ROMEO_AND_JULIET_PROLOGUE).addTag("a").addTag("b").build();
ComplexDocument yiddishDocument = ComplexDocument.newBuilder().setGroup(1).setDocId(1944L).setText(TextSamples.YIDDISH).addTag("c").build();
try (FDBRecordContext context = openContext()) {
openRecordStore(context, metaDataBuilder -> metaDataBuilder.addIndex(COMPLEX_DOC, COMPLEX_TEXT_BY_GROUP));
recordStore.saveRecord(complexDocument);
int firstKeys = getSaveIndexKeyCount(recordStore);
assertEquals(complexDocument.getText().split(" ").length, firstKeys);
recordStore.saveRecord(shakespeareDocument);
int secondKeys = getSaveIndexKeyCount(recordStore) - firstKeys;
assertEquals(82, secondKeys);
recordStore.saveRecord(yiddishDocument);
int thirdKeys = getSaveIndexKeyCount(recordStore) - secondKeys - firstKeys;
assertEquals(9, thirdKeys);
List<Map.Entry<Tuple, List<Integer>>> entryList = scanMapEntries(recordStore, COMPLEX_TEXT_BY_GROUP, Tuple.from(0L, "to"));
assertEquals(Arrays.asList(entryOf(Tuple.from(1066L), Collections.singletonList(3)), entryOf(Tuple.from(1623L), Arrays.asList(18, 108))), entryList);
List<Message> recordList = recordStore.scanIndexRecords(COMPLEX_TEXT_BY_GROUP.getName(), BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(0L, "to")), null, ScanProperties.FORWARD_SCAN).map(FDBIndexedRecord::getRecord).asList().get();
assertEquals(Arrays.asList(complexDocument, shakespeareDocument), recordList);
entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.prefixedBy("א").prepend(Tuple.from(1L))), null);
assertEquals(Arrays.asList(entryOf(Tuple.from(1L, "א", 1944L), Arrays.asList(0, 3)), entryOf(Tuple.from(1L, "און", 1944L), Collections.singletonList(8)), entryOf(Tuple.from(1L, "איז", 1944L), Collections.singletonList(2)), entryOf(Tuple.from(1L, "אן", 1944L), Collections.singletonList(6)), entryOf(Tuple.from(1L, "ארמיי", 1944L), Collections.singletonList(7))), entryList);
// Read the whole store and make sure the values come back in a somewhat sensible way
entryList = toMapEntries(scanIndex(recordStore, COMPLEX_TEXT_BY_GROUP, TupleRange.ALL), null);
assertEquals(firstKeys + secondKeys + thirdKeys, entryList.size());
int i = 0;
String last = null;
for (Map.Entry<Tuple, List<Integer>> entry : entryList) {
assertEquals(3, entry.getKey().size());
if (i < firstKeys + secondKeys) {
assertEquals(0L, entry.getKey().getLong(0));
assertThat(entry.getKey().getLong(2), anyOf(is(1066L), is(1623L)));
} else {
assertEquals(1L, entry.getKey().getLong(0));
assertEquals(1944L, entry.getKey().getLong(2));
if (i == firstKeys + secondKeys) {
last = null;
}
}
if (last != null) {
assertThat(entry.getKey().getString(1), greaterThanOrEqualTo(last));
}
last = entry.getKey().getString(1);
i++;
}
commit(context);
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.
the class RankIndexTest method checkRankScan.
@Test
public void checkRankScan() throws Exception {
TupleRange range = new TupleRange(Tuple.from(0L), Tuple.from(2L), EndpointType.RANGE_INCLUSIVE, EndpointType.RANGE_EXCLUSIVE);
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
int i = 0;
try (RecordCursorIterator<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("BasicRankedRecord$score", IndexScanType.BY_RANK, range, null, ScanProperties.FORWARD_SCAN).asIterator()) {
while (cursor.hasNext()) {
FDBIndexedRecord<Message> rec = cursor.next();
TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
myrec.mergeFrom(rec.getRecord());
assertTrue(myrec.getScore() < 200);
i++;
}
}
assertEquals(2, i);
}
range = TupleRange.allOf(Tuple.from("M", 0L));
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
try (RecordCursor<FDBIndexedRecord<Message>> cursor = recordStore.scanIndexRecords("rank_by_gender", IndexScanType.BY_RANK, range, null, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(1).build()))) {
FDBIndexedRecord<Message> rec = cursor.getNext().get();
TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
myrec.mergeFrom(rec.getRecord());
assertEquals("hector", myrec.getName());
assertEquals(75, myrec.getScore());
}
}
}
use of com.apple.foundationdb.record.provider.foundationdb.FDBIndexedRecord in project fdb-record-layer by FoundationDB.
the class GeophileQueryTest method testJoin.
@Test
@Tag(Tags.Slow)
public // So really this just shows that join results aren't wildly off.
void testJoin() throws Exception {
final RecordMetaDataHook hook = md -> {
md.setSplitLongRecords(true);
md.addIndex("City", CITY_LOCATION_INDEX);
md.addIndex("Country", COUNTRY_SHAPE_INDEX);
};
loadCities(hook, 500000);
loadCountries(hook, 500000);
loadCountryShapes(hook);
final GeophileSpatialIndexJoinPlan plan = new GeophileSpatialIndexJoinPlan("City$location", ScanComparisons.EMPTY, "Country$shape", ScanComparisons.EMPTY);
int[] stats = new int[4];
try (FDBRecordContext context = openContext()) {
openRecordStore(context, hook);
RecordCursor<Pair<FDBIndexedRecord<Message>, FDBIndexedRecord<Message>>> recordCursor = plan.execute(recordStore, EvaluationContext.EMPTY);
final GeometryFactory geometryFactory = new GeometryFactory();
final GeoJsonReader geoJsonReader = new GeoJsonReader(geometryFactory);
recordCursor.forEach(pair -> {
TestRecordsGeoProto.City.Builder cityBuilder = TestRecordsGeoProto.City.newBuilder().mergeFrom(pair.getLeft().getRecord());
TestRecordsGeoProto.Country.Builder countryBuilder = TestRecordsGeoProto.Country.newBuilder().mergeFrom(pair.getRight().getRecord());
Point cityLocation = geometryFactory.createPoint(new Coordinate(cityBuilder.getLocation().getLatitude(), cityBuilder.getLocation().getLongitude()));
Geometry countryShape;
try {
countryShape = GeophileSpatial.swapLatLong(geoJsonReader.read(countryBuilder.getShape()));
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
// A real join would need to do this because of Z-order join false positives.
boolean contained;
try {
contained = countryShape.contains(cityLocation);
} catch (TopologyException ex) {
stats[3]++;
return;
}
if (!contained) {
stats[2]++;
} else if (!countryBuilder.getCode().equals(cityBuilder.getCountry())) {
LOGGER.warn(String.format("Code does not match: %s for %s <> %s for %s", countryBuilder.getCode(), countryBuilder.getName(), cityBuilder.getCountry(), cityBuilder.getName()));
stats[1]++;
} else {
LOGGER.debug(String.format("%s: %s", countryBuilder.getName(), cityBuilder.getName()));
stats[0]++;
}
}).join();
commit(context);
} catch (CompletionException ex) {
// transaction_too_old
assertThat(Throwables.getRootCause(ex), allOf(Matchers.instanceOf(FDBException.class), hasProperty("code", equalTo(1007))));
} finally {
LOGGER.info(String.format("match = %d, no match = %d, no overlap = %d, invalid geometry = %d", stats[0], stats[1], stats[2], stats[3]));
}
}
Aggregations