Examples with ScanProperties com.apple.foundationdb.record.ScanProperties used on opensource projects

Example 16 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreIndexTest method testIndexMissingValidation.

@Test
public void testIndexMissingValidation() throws Exception {
    final int nRecords = 10;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        for (int i = 0; i < nRecords; i++) {
            TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
            recBuilder.setRecNo(i);
            recBuilder.setStrValueIndexed(Integer.toString(i));
            // nRecords is not larger than 10, so the indexes (sorted by the string version of recNo) are in the
            // same order as the records. This can make the test easy.
            recordStore.saveRecord(recBuilder.build());
        }
        commit(context);
    }
    // Delete the indexes of some records.
    Set<InvalidIndexEntry> expectedInvalidEntries = new HashSet<>();
    try (FDBRecordContext context = openContext()) {
        final Index index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        openSimpleRecordStore(context);
        List<FDBStoredRecord<Message>> savedRecords = recordStore.scanRecords(TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        List<IndexEntry> indexEntries = recordStore.scanIndex(index, IndexScanType.BY_VALUE, TupleRange.ALL, null, ScanProperties.FORWARD_SCAN).asList().get();
        for (int i = 0; i < nRecords; i += 2) {
            IndexEntry indexEntry = indexEntries.get(i);
            FDBStoredRecord<Message> record = savedRecords.get(i);
            final Tuple valueKey = indexEntry.getKey();
            final Tuple entryKey = indexEntryKey(index, valueKey, record.getPrimaryKey());
            final byte[] keyBytes = recordStore.indexSubspace(index).pack(valueKey);
            byte[] v0 = recordStore.getContext().ensureActive().get(keyBytes).get();
            recordStore.getContext().ensureActive().clear(keyBytes);
            byte[] v = recordStore.getContext().ensureActive().get(keyBytes).get();
            expectedInvalidEntries.add(InvalidIndexEntry.newMissing(indexEntry, record));
        }
        commit(context);
    }
    try (FDBDatabaseRunner runner = fdb.newRunner()) {
        AtomicInteger generatorCount = new AtomicInteger();
        // Set a scanned records limit to mock when the NoNextReason is out of band.
        RecordCursorIterator<InvalidIndexEntry> cursor = new AutoContinuingCursor<>(runner, (context, continuation) -> new LazyCursor<>(FDBRecordStore.newBuilder().setContext(context).setKeySpacePath(path).setMetaDataProvider(simpleMetaData(NO_HOOK)).openAsync().thenApply(currentRecordStore -> {
            generatorCount.getAndIncrement();
            final Index index = currentRecordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
            ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(Integer.MAX_VALUE).setIsolationLevel(IsolationLevel.SNAPSHOT).setScannedRecordsLimit(4).build());
            return currentRecordStore.getIndexMaintainer(index).validateEntries(continuation, scanProperties);
        }))).asIterator();
        Set<InvalidIndexEntry> results = new HashSet<>();
        cursor.forEachRemaining(results::add);
        assertEquals(expectedInvalidEntries, results);
        // The number of scans is about the number of index entries (orphan validation) plus the number of records
        // (missing validation).
        assertThat(generatorCount.get(), greaterThanOrEqualTo((5 + 10) / 4));
    }
}

Example 17 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanWithSkip.

public void scanWithSkip(@Nonnull Index index, @Nonnull String token, int skip, int limit, boolean reverse) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        final List<IndexEntry> fullResults = scanIndex(recordStore, index, TupleRange.allOf(Tuple.from(token)));
        validateSorted(fullResults);
        final ScanProperties scanProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(limit).setSkip(skip).build().asScanProperties(reverse);
        final RecordCursor<IndexEntry> cursor = recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), null, scanProperties);
        List<IndexEntry> scanResults = cursor.asList().get();
        RecordCursorResult<IndexEntry> noNextResult = cursor.getNext();
        assertThat(noNextResult.hasNext(), is(false));
        assertEquals((limit != ReadTransaction.ROW_LIMIT_UNLIMITED && scanResults.size() == limit) ? RETURN_LIMIT_REACHED : SOURCE_EXHAUSTED, noNextResult.getNoNextReason());
        List<IndexEntry> expectedResults;
        if (reverse) {
            scanResults = new ArrayList<>(scanResults);
            Collections.reverse(scanResults);
            expectedResults = fullResults.subList((limit == ReadTransaction.ROW_LIMIT_UNLIMITED || limit == Integer.MAX_VALUE) ? 0 : Math.max(0, fullResults.size() - skip - limit), Math.max(0, fullResults.size() - skip));
        } else {
            expectedResults = fullResults.subList(Math.min(fullResults.size(), skip), (limit == ReadTransaction.ROW_LIMIT_UNLIMITED || limit == Integer.MAX_VALUE) ? fullResults.size() : Math.min(fullResults.size(), skip + limit));
        }
        assertEquals(expectedResults, scanResults);
    }
}

Example 18 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanMultipleWithScanRecordLimits.

private void scanMultipleWithScanRecordLimits(@Nonnull Index index, @Nonnull List<String> tokens, int scanRecordLimit, boolean reverse) throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        ScanProperties scanProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanRecordLimit).build().asScanProperties(reverse);
        List<RecordCursor<IndexEntry>> cursors = tokens.stream().map(token -> recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), null, scanProperties)).collect(Collectors.toList());
        int cursorIndex = 0;
        int retrieved = 0;
        while (!cursors.isEmpty()) {
            RecordCursor<IndexEntry> cursor = cursors.get(cursorIndex);
            RecordCursorResult<IndexEntry> result = cursor.getNext();
            if (result.hasNext()) {
                retrieved++;
                cursorIndex = (cursorIndex + 1) % cursors.size();
            } else {
                if (!result.getNoNextReason().isSourceExhausted()) {
                    assertEquals(SCAN_LIMIT_REACHED, result.getNoNextReason());
                }
                cursors.remove(cursorIndex);
                if (cursorIndex == cursors.size()) {
                    cursorIndex = 0;
                }
            }
        }
        // With the order that they are retrieved, the maximum value is the scanRecordLimit
        // or the number of tokens.
        assertThat(retrieved, lessThanOrEqualTo(Math.max(scanRecordLimit, tokens.size())));
    }
}

Example 19 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanIndex.

@Nonnull
private static List<IndexEntry> scanIndex(@Nonnull FDBRecordStore store, @Nonnull Index index, @Nonnull TupleRange range) throws ExecutionException, InterruptedException {
    List<IndexEntry> results = scanIndex(store, index, range, ScanProperties.FORWARD_SCAN);
    validateSorted(results);
    List<IndexEntry> backwardResults = new ArrayList<>(scanIndex(store, index, range, ScanProperties.REVERSE_SCAN));
    Collections.reverse(backwardResults);
    assertEquals(results, backwardResults);
    // Validate that
    final int limit = 3;
    for (int i = 0; i < 8; i++) {
        ExecuteProperties.Builder propertiesBuilder = ExecuteProperties.newBuilder();
        if (i < 4) {
            propertiesBuilder.setReturnedRowLimit(limit);
        } else {
            propertiesBuilder.setScannedRecordsLimit(limit);
        }
        List<IndexEntry> paginatedResults = new ArrayList<>(results.size());
        boolean done = false;
        byte[] continuation = null;
        do {
            if (i >= 2 && i < 4) {
                // Use skip instead of continuation to achieve the same results.
                continuation = null;
                propertiesBuilder.setSkip(paginatedResults.size());
            }
            ScanProperties scanProperties = propertiesBuilder.build().asScanProperties(i % 2 == 0);
            int retrieved = 0;
            RecordCursorIterator<IndexEntry> cursor = store.scanIndex(index, BY_TEXT_TOKEN, range, continuation, scanProperties).asIterator();
            while (cursor.hasNext()) {
                paginatedResults.add(cursor.next());
                retrieved++;
            }
            if (done) {
                assertEquals(0, retrieved);
                assertNull(cursor.getContinuation());
            }
            if (retrieved < limit) {
                assertEquals(SOURCE_EXHAUSTED, cursor.getNoNextReason());
            } else {
                assertEquals(limit, retrieved);
                assertEquals(i < 4 ? RETURN_LIMIT_REACHED : SCAN_LIMIT_REACHED, cursor.getNoNextReason());
            }
            done = cursor.getNoNextReason().isSourceExhausted();
            continuation = cursor.getContinuation();
        } while (continuation != null);
        if (i % 2 == 0) {
            Collections.reverse(paginatedResults);
        }
        assertEquals(results, paginatedResults);
    }
    return results;
}

Example 20 with ScanProperties

use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.

the class UnionIntersectionTest method indexScansByPrimaryKey.

/**
 * Create cursors that correspond to union or intersection query and validate that using the custom comparison
 * key works.
 */
@Test
public void indexScansByPrimaryKey() throws Exception {
    final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN;
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        final Index strValueIndex = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
        final Index numValue3Index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$num_value_3_indexed");
        List<Long> recNos = IntersectionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
        assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0).filter(i -> i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
        assertDiscardedAtMost(50, context);
        recNos = UnionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
        assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0 || i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
        assertDiscardedAtMost(83, context);
        commit(context);
    }
}