use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class TextIndexTest method scanMultipleWithScanRecordLimits.
private void scanMultipleWithScanRecordLimits(@Nonnull Index index, @Nonnull List<String> tokens, int scanRecordLimit, boolean reverse) throws Exception {
try (FDBRecordContext context = openContext()) {
openRecordStore(context);
ScanProperties scanProperties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanRecordLimit).build().asScanProperties(reverse);
List<RecordCursor<IndexEntry>> cursors = tokens.stream().map(token -> recordStore.scanIndex(index, BY_TEXT_TOKEN, TupleRange.allOf(Tuple.from(token)), null, scanProperties)).collect(Collectors.toList());
int cursorIndex = 0;
int retrieved = 0;
while (!cursors.isEmpty()) {
RecordCursor<IndexEntry> cursor = cursors.get(cursorIndex);
RecordCursorResult<IndexEntry> result = cursor.getNext();
if (result.hasNext()) {
retrieved++;
cursorIndex = (cursorIndex + 1) % cursors.size();
} else {
if (!result.getNoNextReason().isSourceExhausted()) {
assertEquals(SCAN_LIMIT_REACHED, result.getNoNextReason());
}
cursors.remove(cursorIndex);
if (cursorIndex == cursors.size()) {
cursorIndex = 0;
}
}
}
// With the order that they are retrieved, the maximum value is the scanRecordLimit
// or the number of tokens.
assertThat(retrieved, lessThanOrEqualTo(Math.max(scanRecordLimit, tokens.size())));
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class TextIndexTest method scanIndex.
@Nonnull
private static List<IndexEntry> scanIndex(@Nonnull FDBRecordStore store, @Nonnull Index index, @Nonnull TupleRange range) throws ExecutionException, InterruptedException {
List<IndexEntry> results = scanIndex(store, index, range, ScanProperties.FORWARD_SCAN);
validateSorted(results);
List<IndexEntry> backwardResults = new ArrayList<>(scanIndex(store, index, range, ScanProperties.REVERSE_SCAN));
Collections.reverse(backwardResults);
assertEquals(results, backwardResults);
// Validate that
final int limit = 3;
for (int i = 0; i < 8; i++) {
ExecuteProperties.Builder propertiesBuilder = ExecuteProperties.newBuilder();
if (i < 4) {
propertiesBuilder.setReturnedRowLimit(limit);
} else {
propertiesBuilder.setScannedRecordsLimit(limit);
}
List<IndexEntry> paginatedResults = new ArrayList<>(results.size());
boolean done = false;
byte[] continuation = null;
do {
if (i >= 2 && i < 4) {
// Use skip instead of continuation to achieve the same results.
continuation = null;
propertiesBuilder.setSkip(paginatedResults.size());
}
ScanProperties scanProperties = propertiesBuilder.build().asScanProperties(i % 2 == 0);
int retrieved = 0;
RecordCursorIterator<IndexEntry> cursor = store.scanIndex(index, BY_TEXT_TOKEN, range, continuation, scanProperties).asIterator();
while (cursor.hasNext()) {
paginatedResults.add(cursor.next());
retrieved++;
}
if (done) {
assertEquals(0, retrieved);
assertNull(cursor.getContinuation());
}
if (retrieved < limit) {
assertEquals(SOURCE_EXHAUSTED, cursor.getNoNextReason());
} else {
assertEquals(limit, retrieved);
assertEquals(i < 4 ? RETURN_LIMIT_REACHED : SCAN_LIMIT_REACHED, cursor.getNoNextReason());
}
done = cursor.getNoNextReason().isSourceExhausted();
continuation = cursor.getContinuation();
} while (continuation != null);
if (i % 2 == 0) {
Collections.reverse(paginatedResults);
}
assertEquals(results, paginatedResults);
}
return results;
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class VersionIndexTest method assertMaxVersionWithExtraColumn.
@SuppressWarnings("try")
private void assertMaxVersionWithExtraColumn(int column, @Nonnull FDBRecordVersion recordVersion) {
try (FDBRecordContext context = openContext(maxEverVersionWithExtraColumnHook)) {
Index index = metaData.getIndex("max_ever_version_with_extra_column");
IndexEntry entry = new IndexEntry(index, Key.Evaluated.EMPTY, Key.Evaluated.concatenate(column, recordVersion));
assertMaxVersionEntries(index, Collections.singletonList(entry));
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class BitmapValueIndexTest method andOrQueryWithContinuation.
@Test
void andOrQueryWithContinuation() {
try (FDBRecordContext context = openContext()) {
createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
saveRecords(100, 200);
commit(context);
}
try (FDBRecordContext context = openContext()) {
createOrOpenRecordStore(context, metaData(REC_NO_BY_STR_NUMS_HOOK));
setupPlanner(null);
final RecordQueryPlan queryPlan = plan(BITMAP_VALUE_REC_NO_BY_STR, Query.and(Query.field("str_value").equalsValue("odd"), Query.field("num_value_2").equalsValue(3), Query.or(Query.field("num_value_3").equalsValue(1), Query.field("num_value_3").equalsValue(4))));
List<Integer> onBits = new ArrayList<>();
int ntimes = 0;
byte[] continuation = null;
do {
RecordCursor<IndexEntry> cursor = queryPlan.execute(recordStore, EvaluationContext.EMPTY, continuation, ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()).map(FDBQueriedRecord::getIndexEntry);
RecordCursorResult<IndexEntry> cursorResult = cursor.forEachResult(i -> onBits.addAll(collectOnBits(i.get()))).join();
ntimes++;
continuation = cursorResult.getContinuation().toBytes();
} while (continuation != null);
assertThat(onBits, equalTo(IntStream.range(100, 200).boxed().filter(i -> (i & 1) == 1).filter(i -> (i % 7) == 3 && ((i % 5) == 1 || (i % 5) == 4)).collect(Collectors.toList())));
assertThat(ntimes, equalTo(4));
}
}
use of com.apple.foundationdb.record.IndexEntry in project fdb-record-layer by FoundationDB.
the class UnionIntersectionTest method indexScansByPrimaryKey.
/**
* Create cursors that correspond to union or intersection query and validate that using the custom comparison
* key works.
*/
@Test
public void indexScansByPrimaryKey() throws Exception {
final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
final Index strValueIndex = recordStore.getRecordMetaData().getIndex("MySimpleRecord$str_value_indexed");
final Index numValue3Index = recordStore.getRecordMetaData().getIndex("MySimpleRecord$num_value_3_indexed");
List<Long> recNos = IntersectionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0).filter(i -> i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
assertDiscardedAtMost(50, context);
recNos = UnionCursor.create((IndexEntry entry) -> TupleHelpers.subTuple(entry.getKey(), 1, entry.getKey().size()).getItems(), false, (byte[] leftContinuation) -> recordStore.scanIndex(strValueIndex, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from("even")), leftContinuation, scanProperties), (byte[] rightContinuation) -> recordStore.scanIndex(numValue3Index, IndexScanType.BY_VALUE, TupleRange.allOf(Tuple.from(1)), rightContinuation, scanProperties), null, recordStore.getTimer()).mapPipelined(indexEntry -> recordStore.loadRecordAsync(strValueIndex.getEntryPrimaryKey(indexEntry.getKey())), recordStore.getPipelineSize(PipelineOperation.INDEX_TO_RECORD)).map(this::storedRecordRecNo).asList().get();
assertEquals(LongStream.range(0, 100).filter(i -> i % 2 == 0 || i % 3 != 0).boxed().collect(Collectors.toList()), recNos);
assertDiscardedAtMost(83, context);
commit(context);
}
}
Aggregations