use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method testIndexMissingValidation.
@Test
public void testIndexMissingValidation() throws Exception {
final int nRecords = 10;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
for (int i = 0; i < nRecords; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(i);
recBuilder.setStrValueIndexed(Integer.toString(i));
// nRecords is not larger than 10, so the indexes (sorted by the string version of recNo) are in the
// same order as the records. This can make the test easy.
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
// Delete the indexes of some records.
Set<InvalidIndexEntry> expectedInvalidEntries = new HashSet<>();
try (FDBRecordContext context = openContext()) {
final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
openSimpleRecordStore(context);
List<FDBStoredRecord<Message>> savedRecords = recordStore.scanRecords(TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
List<IndexEntry> indexEntries = recordStore.scanIndex(index, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
for (int i = 0; i < nRecords; i += 2) {
IndexEntry indexEntry = indexEntries.get(i);
FDBStoredRecord<Message> record = savedRecords.get(i);
final Tuple valueKey = indexEntry.getKey();
final Tuple entryKey = indexEntryKey(index, valueKey, record.getPrimaryKey());
final byte[] keyBytes = recordStore.indexSubspace(index).pack(valueKey);
byte[] v0 = recordStore.getContext().ensureActive().get(keyBytes).get();
recordStore.getContext().ensureActive().clear(keyBytes);
byte[] v = recordStore.getContext().ensureActive().get(keyBytes).get();
expectedInvalidEntries.add(InvalidIndexEntry.newMissing(indexEntry, record));
}
commit(context);
}
try (FDBDatabaseRunner runner = fdb.newRunner()) {
AtomicInteger generatorCount = new AtomicInteger();
// Set a scanned records limit to mock when the NoNextReason is out of band.
RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).openAsync().thenApply(currentRecordStore -> {
generatorCount.getAndIncrement();
final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
}))).asIterator();
Set<InvalidIndexEntry> results = new HashSet<>();
cursor.forEachRemaining(results::add);
assertEquals(expectedInvalidEntries, results);
// The number of scans is about the number of index entries (orphan validation) plus the number of records
// (missing validation).
assertThat(generatorCount.get(), greaterThanOrEqualTo((5 + 10) / 4));
}
}
use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method uniquenessViolationsWithFanOut.
@Test
public void uniquenessViolationsWithFanOut() throws Exception {
Index index = new Index("MySimpleRecord$repeater", field("repeater", FanType.FanOut), Index.EMPTY_VALUE, IndexTypes.VALUE, IndexOptions.UNIQUE_OPTIONS);
RecordMetaDataHook hook = metaData -> metaData.addIndex("MySimpleRecord", index);
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertThat(recordStore.isIndexWriteOnly(index), is(false));
recordStore.markIndexWriteOnly(index).get();
assertThat(recordStore.isIndexWriteOnly(index), is(true));
context.commit();
}
TestRecords1Proto.MySimpleRecord record1 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1066L).addRepeater(1).addRepeater(2).addRepeater(3).build();
TestRecords1Proto.MySimpleRecord record2 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1793L).addRepeater(2).addRepeater(3).build();
TestRecords1Proto.MySimpleRecord record3 = TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1849L).addRepeater(3).build();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.saveRecord(record1);
recordStore.saveRecord(record2);
recordStore.saveRecord(record3);
context.commit();
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
assertEquals(5, (int) recordStore.scanUniquenessViolations(index).getCount().get());
assertEquals(0, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(1)).getCount().get());
assertEquals(2, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).getCount().get());
assertEquals(3, (int) recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).getCount().get());
RecordCursorIterator<RecordIndexUniquenessViolation> cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(3)).asIterator();
assertTrue(cursor.hasNext());
RecordIndexUniquenessViolation next = cursor.next();
assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
assertEquals(Tuple.from(1066L), next.getPrimaryKey());
assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1793L), Tuple.from(1849L))));
assertTrue(cursor.hasNext());
next = cursor.next();
assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
assertEquals(Tuple.from(1793L), next.getPrimaryKey());
assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1849L))));
assertTrue(cursor.hasNext());
next = cursor.next();
assertEquals(Tuple.from(3L), next.getIndexEntry().getKey());
assertEquals(Tuple.from(1849L), next.getPrimaryKey());
assertThat(next.getExistingKey(), is(oneOf(Tuple.from(1066L), Tuple.from(1793L))));
assertFalse(cursor.hasNext());
cursor = recordStore.scanUniquenessViolations(index, Key.Evaluated.scalar(2)).asIterator();
assertTrue(cursor.hasNext());
next = cursor.next();
assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
assertEquals(Tuple.from(1066L), next.getPrimaryKey());
assertEquals(Tuple.from(1793L), next.getExistingKey());
assertTrue(cursor.hasNext());
next = cursor.next();
assertEquals(Tuple.from(2L), next.getIndexEntry().getKey());
assertEquals(Tuple.from(1793L), next.getPrimaryKey());
assertEquals(Tuple.from(1066L), next.getExistingKey());
assertFalse(cursor.hasNext());
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.resolveUniquenessViolation(index, Key.Evaluated.scalar(3), null).get();
assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
assertNull(recordStore.loadRecord(Tuple.from(1066L)));
assertNull(recordStore.loadRecord(Tuple.from(1793L)));
assertNull(recordStore.loadRecord(Tuple.from(1849L)));
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1066L)).get();
assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
assertNotNull(recordStore.loadRecord(Tuple.from(1066L)));
assertNull(recordStore.loadRecord(Tuple.from(1793L)));
assertNull(recordStore.loadRecord(Tuple.from(1849L)));
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1793L)).get();
assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
assertNull(recordStore.loadRecord(Tuple.from(1066L)));
assertNotNull(recordStore.loadRecord(Tuple.from(1793L)));
assertNull(recordStore.loadRecord(Tuple.from(1849L)));
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
recordStore.resolveUniquenessViolation(index, Tuple.from(3), Tuple.from(1849L)).get();
assertEquals(0, (int) recordStore.scanUniquenessViolations(index).getCount().get());
assertNull(recordStore.loadRecord(Tuple.from(1066L)));
assertNull(recordStore.loadRecord(Tuple.from(1793L)));
assertNotNull(recordStore.loadRecord(Tuple.from(1849L)));
}
}
use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method queryOnVersion.
@ParameterizedTest(name = "queryOnVersion [formatVersion = {0}, splitLongRecords = {1}]")
@MethodSource("formatVersionArguments")
@SuppressWarnings("try")
public void queryOnVersion(int testFormatVersion, boolean testSplitLongRecords) {
formatVersion = testFormatVersion;
splitLongRecords = testSplitLongRecords;
List<MySimpleRecord> simpleRecords = IntStream.range(0, 30).mapToObj(id -> MySimpleRecord.newBuilder().setRecNo(id * 2).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
List<TestRecords1Proto.MyOtherRecord> otherRecords = IntStream.range(0, 30).mapToObj(id -> TestRecords1Proto.MyOtherRecord.newBuilder().setRecNo(id * 2 + 1).setNumValue2(id % 2).setNumValue3Indexed(id % 3).build()).collect(Collectors.toList());
Iterator<MySimpleRecord> simpleIterator = simpleRecords.iterator();
Iterator<TestRecords1Proto.MyOtherRecord> otherIterator = otherRecords.iterator();
while (simpleIterator.hasNext()) {
try (FDBRecordContext context = openContext(simpleVersionHook)) {
int done = 0;
while (simpleIterator.hasNext() && done != 5) {
recordStore.saveRecord(simpleIterator.next());
recordStore.saveRecord(otherIterator.next());
done += 1;
}
context.commit();
}
}
// Query all records.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = Stream.concat(simpleRecords.stream().map(MySimpleRecord::getRecNo), otherRecords.stream().map(TestRecords1Proto.MyOtherRecord::getRecNo)).sorted().collect(Collectors.toList());
FDBRecordVersion last = null;
List<Long> receivedKeys = new ArrayList<>();
int totalSeen = 0;
while (true) {
RecordQueryPlan plan;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion <,>)", plan.toString());
} else {
RecordQuery query = RecordQuery.newBuilder().setFilter(Query.version().greaterThan(last)).setSort(VersionKeyExpression.VERSION).build();
plan = planner.plan(query);
assertEquals("Index(globalVersion ([" + last.toVersionstamp() + "],>)", plan.toString());
}
RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asIterator();
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBQueriedRecord<Message> record = cursor.next();
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(field("rec_no").evaluateSingleton(record.getStoredRecord()).toTuple().getLong(0));
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() + otherRecords.size(), totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query MySimpleRecord based on value.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_2").equalsValue(0)).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]])", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(3).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals((simpleRecords.size() + 1) / 2, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that requires also filtering
try (FDBRecordContext context = openContext(simpleVersionHook)) {
List<Long> expectedKeys = simpleRecords.stream().filter(rec -> rec.getNumValue2() == 0 && rec.getNumValue3Indexed() == 0).map(MySimpleRecord::getRecNo).collect(Collectors.toList());
List<Long> receivedKeys = new ArrayList<>();
FDBRecordVersion last = null;
int totalSeen = 0;
while (true) {
RecordCursorIterator<? extends FDBRecord<Message>> cursor;
if (last == null) {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version [[0],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
} else {
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.and(Query.field("num_value_2").equalsValue(0), Query.field("num_value_3_indexed").equalsValue(0), Query.version().greaterThan(last))).setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num2-version ([0, " + last.toVersionstamp() + "],[0]]) | num_value_3_indexed EQUALS 0", plan.toString());
cursor = recordStore.executeQuery(plan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).asIterator();
}
boolean hasAny = false;
while (cursor.hasNext()) {
hasAny = true;
FDBRecord<Message> record = cursor.next();
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertEquals(0, simpleRecord.getNumValue2());
assertTrue(record.hasVersion());
if (last != null) {
assertThat(last, lessThan(record.getVersion()));
}
last = record.getVersion();
receivedKeys.add(simpleRecord.getRecNo());
totalSeen += 1;
}
if (!hasAny) {
break;
}
}
assertEquals(simpleRecords.size() / 6, totalSeen);
assertEquals(expectedKeys, receivedKeys);
}
// Query that can't be satisfied with an index scan.
try (FDBRecordContext context = openContext(simpleVersionHook)) {
// Preliminary query to get a read version.
RecordQuery prelimQuery = RecordQuery.newBuilder().setSort(VersionKeyExpression.VERSION).build();
RecordQueryPlan prelimPlan = planner.plan(prelimQuery);
FDBRecordVersion chosenVersion = recordStore.executeQuery(prelimPlan, null, ExecuteProperties.newBuilder().setReturnedRowLimit(10).build()).asList().thenApply(list -> list.get(list.size() - 1).getVersion()).join();
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.version().greaterThan(chosenVersion)).setSort(field("num_value_3_indexed")).build();
RecordQueryPlan plan = planner.plan(query);
assertEquals("Index(MySimpleRecord$num_value_3_indexed <,>) | version GREATER_THAN " + chosenVersion.toString(), plan.toString());
List<FDBQueriedRecord<Message>> records = recordStore.executeQuery(plan).asList().join();
int last = -1;
for (FDBQueriedRecord<Message> record : records) {
MySimpleRecord simpleRecord = MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).build();
assertThat(last, lessThanOrEqualTo(simpleRecord.getNumValue3Indexed()));
assertTrue(record.hasVersion());
assertThat(chosenVersion, lessThan(record.getVersion()));
last = simpleRecord.getNumValue3Indexed();
}
assertEquals(simpleRecords.size() - 5, records.size());
}
}
use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method validateUsingOlderVersionFormat.
private <M extends Message> void validateUsingOlderVersionFormat(@Nonnull List<FDBStoredRecord<M>> storedRecords) {
// Make sure all of the records have versions in the old keyspace
final Subspace legacyVersionSubspace = recordStore.getLegacyVersionSubspace();
RecordCursorIterator<Pair<Tuple, FDBRecordVersion>> versionKeyPairs = KeyValueCursor.Builder.withSubspace(legacyVersionSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().map(kv -> Pair.of(legacyVersionSubspace.unpack(kv.getKey()), FDBRecordVersion.fromBytes(kv.getValue()))).asIterator();
for (FDBStoredRecord<M> storedRecord : storedRecords) {
assertTrue(versionKeyPairs.hasNext());
Pair<Tuple, FDBRecordVersion> versionPair = versionKeyPairs.next();
assertEquals(storedRecord.getPrimaryKey(), versionPair.getLeft());
assertEquals(storedRecord.getVersion(), versionPair.getRight());
}
assertFalse(versionKeyPairs.hasNext());
// Validate that no value in the record subspace begins with the type code for versionstamps
final Subspace recordsSubspace = recordStore.recordsSubspace();
KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(recordStore.getRecordContext()).setScanProperties(ScanProperties.FORWARD_SCAN).build().forEach(kv -> assertNotEquals(VERSIONSTAMP_CODE, kv.getValue()[0])).join();
}
use of com.apple.foundationdb.record.RecordCursorIterator in project fdb-record-layer by FoundationDB.
the class RankIndexTest method queryWithRanks.
@Test
public void queryWithRanks() throws Exception {
RecordQuery query = RecordQuery.newBuilder().setRecordType("BasicRankedRecord").setFilter(Query.field("gender").equalsValue("M")).setSort(Key.Expressions.field("score")).build();
RecordQueryPlan plan = planner.plan(query);
QueryRecordFunction<Long> ranker = Query.rank(Key.Expressions.field("score").groupBy(Key.Expressions.field("gender")));
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
try (RecordCursorIterator<? extends Pair<Message, Long>> cursor = recordStore.executeQuery(plan).mapPipelined(record -> ranker.eval(recordStore, EvaluationContext.EMPTY, record.getStoredRecord()).thenApply(rank -> new ImmutablePair<>(record.getRecord(), rank)), recordStore.getPipelineSize(PipelineOperation.RECORD_FUNCTION)).asIterator()) {
long rank = 0;
while (cursor.hasNext()) {
Pair<Message, Long> recWithRank = cursor.next();
TestRecordsRankProto.BasicRankedRecord.Builder myrec = TestRecordsRankProto.BasicRankedRecord.newBuilder();
myrec.mergeFrom(recWithRank.getLeft());
assertEquals((Long) rank++, recWithRank.getRight());
}
}
}
}
Aggregations