Examples with LazyCursor com.apple.foundationdb.record.cursors.LazyCursor used on opensource projects

Example 1 with LazyCursor

use of com.apple.foundationdb.record.cursors.LazyCursor in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectory method listSubdirectoryAsync.

@Nonnull
// SonarQube doesn't realize that the cursor is wrapped and returned
@SuppressWarnings("squid:S2095")
protected RecordCursor<ResolvedKeySpacePath> listSubdirectoryAsync(@Nullable KeySpacePath listFrom, @Nonnull FDBRecordContext context, @Nonnull String subdirName, @Nullable ValueRange<?> valueRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (listFrom != null && listFrom.getDirectory() != this) {
        throw new RecordCoreException("Provided path does not belong to this directory").addLogInfo("path", listFrom, "directory", this.getName());
    }
    final KeySpaceDirectory subdir = getSubdirectory(subdirName);
    final CompletableFuture<ResolvedKeySpacePath> resolvedFromFuture = listFrom == null ? CompletableFuture.completedFuture(null) : listFrom.toResolvedPathAsync(context);
    // The chained cursor cannot implement reverse scan, so we implement it by having the
    // inner key value cursor do the reversing but telling the chained cursor we are moving
    // forward.
    final ScanProperties chainedCursorScanProperties;
    if (scanProperties.isReverse()) {
        chainedCursorScanProperties = scanProperties.setReverse(false);
    } else {
        chainedCursorScanProperties = scanProperties;
    }
    // For the read of the individual row keys, we only want to read a single key. In addition,
    // the ChainedCursor is going to do counting of our reads to apply any limits that were specified
    // on the ScanProperties.  We don't want the inner KeyValueCursor in nextTuple() to ALSO count those
    // same reads so we clear out its limits.
    final ScanProperties keyReadScanProperties = scanProperties.with(props -> props.clearState().setReturnedRowLimit(1));
    return new LazyCursor<>(resolvedFromFuture.thenCompose(resolvedFrom -> {
        final Subspace subspace = resolvedFrom == null ? new Subspace() : resolvedFrom.toSubspace();
        return subdir.getValueRange(context, valueRange, subspace).thenApply(range -> {
            final RecordCursor<Tuple> cursor = new ChainedCursor<>(context, lastKey -> nextTuple(context, subspace, range, lastKey, keyReadScanProperties), Tuple::pack, Tuple::fromBytes, continuation, chainedCursorScanProperties);
            return cursor.mapPipelined(tuple -> {
                final Tuple key = Tuple.fromList(tuple.getItems());
                return findChildForKey(context, resolvedFrom, key, 1, 0);
            }, 1);
        });
    }), context.getExecutor());
}

Example 2 with LazyCursor

use of com.apple.foundationdb.record.cursors.LazyCursor in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testIndexOrphanValidationByAutoContinuingCursor.

@Test
public void testIndexOrphanValidationByAutoContinuingCursor() throws Exception {
    Set<IndexEntry> expectedInvalidEntries = setUpIndexOrphanValidation();
    try (FDBDatabaseRunner runner = fdb.newRunner()) {
        AtomicInteger generatorCount = new AtomicInteger();
        // Set a scanned records limit to mock when the transaction is out of band.
        RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).uncheckedOpenAsync().thenApply(currentRecordStore -> {
            generatorCount.getAndIncrement();
            final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
            ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
            return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
        }))).asIterator();
        Set<IndexEntry> results = new HashSet<>();
        while (cursor.hasNext()) {
            InvalidIndexEntry invalidIndexEntry = cursor.next();
            assertEquals(InvalidIndexEntry.Reasons.ORPHAN, invalidIndexEntry.getReason());
            IndexEntry entry = invalidIndexEntry.getEntry();
            assertFalse(results.contains(entry), "Entry " + entry + " is duplicated");
            results.add(entry);
        }
        assertEquals(expectedInvalidEntries, results, "Validation should return the index entry that has no associated record.");
        // The number of scans is about the number of index entries (orphan validation) plus the number of records
        // (missing validation).
        assertThat(generatorCount.get(), greaterThanOrEqualTo((20 + 10) / 4));
    }
}

Example 3 with LazyCursor

use of com.apple.foundationdb.record.cursors.LazyCursor in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testIndexMissingValidation.

@Test
public void testIndexMissingValidation() throws Exception {
    final int nRecords = 10;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < nRecords; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed(Integer.toString(i));
            // nRecords is not larger than 10, so the indexes (sorted by the string version of recNo) are in the
            // same order as the records. This can make the test easy.
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    // Delete the indexes of some records.
    Set<InvalidIndexEntry> expectedInvalidEntries = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        openSimpleRecordStore(context);
        List<FDBStoredRecord<Message>> savedRecords = recordStore.scanRecords(TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        List<IndexEntry> indexEntries = recordStore.scanIndex(index, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        for (int i = 0; i < nRecords; i += 2) {
            IndexEntry indexEntry = indexEntries.get(i);
            FDBStoredRecord<Message> record = savedRecords.get(i);
            final Tuple valueKey = indexEntry.getKey();
            final Tuple entryKey = indexEntryKey(index, valueKey, record.getPrimaryKey());
            final byte[] keyBytes = recordStore.indexSubspace(index).pack(valueKey);
            byte[] v0 = recordStore.getContext().ensureActive().get(keyBytes).get();
            recordStore.getContext().ensureActive().clear(keyBytes);
            byte[] v = recordStore.getContext().ensureActive().get(keyBytes).get();
            expectedInvalidEntries.add(InvalidIndexEntry.newMissing(indexEntry, record));
        }
        commit(context);
    }
    try (FDBDatabaseRunner runner = fdb.newRunner()) {
        AtomicInteger generatorCount = new AtomicInteger();
        // Set a scanned records limit to mock when the NoNextReason is out of band.
        RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).openAsync().thenApply(currentRecordStore -> {
            generatorCount.getAndIncrement();
            final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
            ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
            return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
        }))).asIterator();
        Set<InvalidIndexEntry> results = new HashSet<>();
        cursor.forEachRemaining(results::add);
        assertEquals(expectedInvalidEntries, results);
        // The number of scans is about the number of index entries (orphan validation) plus the number of records
        // (missing validation).
        assertThat(generatorCount.get(), greaterThanOrEqualTo((5 + 10) / 4));
    }
}