use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class TextIndexTest method toMapEntries.
@SuppressWarnings("unchecked")
@Nonnull
private List<Map.Entry<Tuple, List<Integer>>> toMapEntries(@Nonnull List<IndexEntry> indexEntries, @Nullable Tuple prefix) {
List<Map.Entry<Tuple, List<Integer>>> mapEntries = new ArrayList<>(indexEntries.size());
for (IndexEntry entry : indexEntries) {
List<Integer> positionList = (List<Integer>) entry.getValue().get(0);
if (prefix == null) {
mapEntries.add(entryOf(entry.getKey(), positionList));
} else {
assertEquals(TupleHelpers.subTuple(entry.getKey(), 0, prefix.size()), prefix);
mapEntries.add(entryOf(TupleHelpers.subTuple(entry.getKey(), prefix.size(), entry.getKey().size()), positionList));
}
}
return mapEntries;
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class TextIndexTest method scanWithContinuations.
private void scanWithContinuations(@Nonnull Index index, @Nonnull String token, int limit, boolean reverse) throws Exception {
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
final List<IndexEntry> firstResults = scanIndex(recordStore, index, TupleRange.allOf(Tuple.from(token)));
validateSorted(firstResults);
List<IndexEntry> scanResults = new ArrayList<>(firstResults.size());
byte[] continuation = null;
do {
RecordCursorIterator<IndexEntry> cursor = recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), continuation, reverse ? ScanProperties.REVERSE_SCAN : ScanProperties.FORWARD_SCAN).asIterator();
for (int i = 0; i < limit || limit == 0; i++) {
if (cursor.hasNext()) {
scanResults.add(cursor.next());
} else {
break;
}
}
continuation = cursor.getContinuation();
// fire off but don't wait for an on-has-next
CompletableFuture<Boolean> hasNextFuture = cursor.onHasNext();
// wait for the same on-has-next
assertEquals(hasNextFuture.get(), cursor.hasNext());
if (!cursor.hasNext()) {
assertThat(cursor.getNoNextReason().isSourceExhausted(), is(true));
}
} while (continuation != null);
if (reverse) {
Collections.reverse(scanResults);
}
assertEquals(firstResults, scanResults);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method testIndexOrphanValidationByAutoContinuingCursor.
@Test
public void testIndexOrphanValidationByAutoContinuingCursor() throws Exception {
Set<IndexEntry> expectedInvalidEntries = setUpIndexOrphanValidation();
try (FDBDatabaseRunner runner = fdb.newRunner()) {
AtomicInteger generatorCount = new AtomicInteger();
// Set a scanned records limit to mock when the transaction is out of band.
RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).uncheckedOpenAsync().thenApply(currentRecordStore -> {
generatorCount.getAndIncrement();
final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
}))).asIterator();
Set<IndexEntry> results = new HashSet<>();
while (cursor.hasNext()) {
InvalidIndexEntry invalidIndexEntry = cursor.next();
assertEquals(InvalidIndexEntry.Reasons.ORPHAN, invalidIndexEntry.getReason());
IndexEntry entry = invalidIndexEntry.getEntry();
assertFalse(results.contains(entry), "Entry " + entry + " is duplicated");
results.add(entry);
}
assertEquals(expectedInvalidEntries, results, "Validation should return the index entry that has no associated record.");
// The number of scans is about the number of index entries (orphan validation) plus the number of records
// (missing validation).
assertThat(generatorCount.get(), greaterThanOrEqualTo((20 + 10) / 4));
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method orphanedIndexEntry.
@Test
public void orphanedIndexEntry() throws Exception {
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context);
recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(1).setStrValueIndexed("foo").setNumValueUnique(1).build());
recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(2).setStrValueIndexed("bar").setNumValueUnique(2).build());
recordStore.saveRecord(TestRecords1Proto.MySimpleRecord.newBuilder().setRecNo(3).setStrValueIndexed("baz").setNumValueUnique(3).build());
commit(context);
}
// Delete the "bar" record with the index removed.
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context, builder -> {
builder.removeIndex("MySimpleRecord$str_value_indexed");
});
recordStore.deleteRecord(Tuple.from(2));
commit(context);
}
// associated record.
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context);
// Verify our entries
Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
assertTrue(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("foo", 1), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Foo' should exist");
assertFalse(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("bar", 2), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Bar' should be deleted");
assertTrue(recordStore.hasIndexEntryRecord(new IndexEntry(index, Tuple.from("baz", 3), TupleHelpers.EMPTY), IsolationLevel.SERIALIZABLE).get(), "'Baz' should exist");
try {
recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed").asList().get();
fail("Scan should have found orphaned record");
} catch (ExecutionException e) {
assertEquals("record not found from index entry", e.getCause().getMessage());
}
commit(context);
}
// Try again, but this time scan allowing orphaned entries.
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context);
List<FDBIndexedRecord<Message>> records = recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, TupleRange.ALL, null, IndexOrphanBehavior.RETURN, ScanProperties.FORWARD_SCAN).asList().get();
assertEquals(records.size(), 3);
for (FDBIndexedRecord<Message> record : records) {
if (record.getIndexEntry().getKey().getString(0).equals("bar")) {
assertFalse(record.hasStoredRecord(), "Entry for 'bar' should be orphaned");
assertThrows(RecordCoreException.class, record::getStoredRecord);
assertThrows(RecordCoreException.class, record::getRecord);
} else {
assertTrue(record.hasStoredRecord(), "Entry for '" + record.getIndexEntry().getKey() + "' should have an associated record");
}
}
commit(context);
}
// Try once again, but this time skipping orphaned entries
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context);
List<FDBIndexedRecord<Message>> records = recordStore.scanIndexRecords("MySimpleRecord$str_value_indexed", IndexScanType.BY_VALUE, TupleRange.ALL, null, IndexOrphanBehavior.SKIP, ScanProperties.FORWARD_SCAN).asList().get();
assertEquals(records.size(), 2);
for (FDBIndexedRecord<Message> record : records) {
assertNotEquals("bar", record.getIndexEntry().getKey().getString(0));
assertTrue(record.hasStoredRecord(), "Entry for '" + record.getIndexEntry().getKey() + "' should have an associated record");
}
commit(context);
}
// Validate the index. Should only return the index entry that has no associated record.
try (FDBRecordContext context = openContext()) {
uncheckedOpenSimpleRecordStore(context);
final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
final List<InvalidIndexEntry> invalidIndexEntries = recordStore.getIndexMaintainer(index).validateEntries(null, null).asList().get();
assertEquals(Collections.singletonList(InvalidIndexEntry.newOrphan(new IndexEntry(index, Tuple.from("bar", 2), TupleHelpers.EMPTY))), invalidIndexEntries, "Validation should return the index entry that has no associated record.");
commit(context);
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreIndexTest method testIndexMissingValidation.
@Test
public void testIndexMissingValidation() throws Exception {
final int nRecords = 10;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
for (int i = 0; i < nRecords; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
recBuilder.setRecNo(i);
recBuilder.setStrValueIndexed(Integer.toString(i));
// nRecords is not larger than 10, so the indexes (sorted by the string version of recNo) are in the
// same order as the records. This can make the test easy.
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
// Delete the indexes of some records.
Set<InvalidIndexEntry> expectedInvalidEntries = new HashSet<>();
try (FDBRecordContext context = openContext()) {
final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
openSimpleRecordStore(context);
List<FDBStoredRecord<Message>> savedRecords = recordStore.scanRecords(TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
List<IndexEntry> indexEntries = recordStore.scanIndex(index, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
for (int i = 0; i < nRecords; i += 2) {
IndexEntry indexEntry = indexEntries.get(i);
FDBStoredRecord<Message> record = savedRecords.get(i);
final Tuple valueKey = indexEntry.getKey();
final Tuple entryKey = indexEntryKey(index, valueKey, record.getPrimaryKey());
final byte[] keyBytes = recordStore.indexSubspace(index).pack(valueKey);
byte[] v0 = recordStore.getContext().ensureActive().get(keyBytes).get();
recordStore.getContext().ensureActive().clear(keyBytes);
byte[] v = recordStore.getContext().ensureActive().get(keyBytes).get();
expectedInvalidEntries.add(InvalidIndexEntry.newMissing(indexEntry, record));
}
commit(context);
}
try (FDBDatabaseRunner runner = fdb.newRunner()) {
AtomicInteger generatorCount = new AtomicInteger();
// Set a scanned records limit to mock when the NoNextReason is out of band.
RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).openAsync().thenApply(currentRecordStore -> {
generatorCount.getAndIncrement();
final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
}))).asIterator();
Set<InvalidIndexEntry> results = new HashSet<>();
cursor.forEachRemaining(results::add);
assertEquals(expectedInvalidEntries, results);
// The number of scans is about the number of index entries (orphan validation) plus the number of records
// (missing validation).
assertThat(generatorCount.get(), greaterThanOrEqualTo((5 + 10) / 4));
}
}
Aggregations