use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreByteLimitTest method testPlansReturnSameRecordsRegardlessOfLimit.
@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plans")
public void testPlansReturnSameRecordsRegardlessOfLimit(String description, boolean notUsed, RecordQueryPlan plan) throws Exception {
setupSimpleRecordStore();
final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
record.mergeFrom(r.getRecord());
return record.getRecNo();
};
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
final List<Long> allAtOnce;
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
allAtOnce = cursor.map(getRecNo).asList().get();
}
for (long byteLimit = 0; byteLimit < 1000; byteLimit += 100) {
final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(byteLimit).build();
final List<Long> byContinuation = new ArrayList<>(allAtOnce.size());
byte[] continuation = null;
do {
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties)) {
RecordCursorResult<Long> result;
do {
result = cursor.onNext().get().map(getRecNo);
if (result.hasNext()) {
byContinuation.add(result.get());
}
} while (result.hasNext());
continuation = result.getContinuation().toBytes();
}
} while (continuation != null);
assertEquals(allAtOnce, byContinuation);
}
}
}
use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreByteLimitTest method queryWithWideOrOfFullTextPrefixPredicates.
/**
* Queries with an OR of {@link com.apple.foundationdb.record.query.expressions.Text#containsPrefix(String)}
* predicates get planned as {@link com.apple.foundationdb.record.query.plan.plans.RecordQueryUnorderedUnionPlan}s,
* which have unusual semantics where results are returned in an undefined order as soon as any child has one.
* Therefore, the assertions made in {@link #assertPlanLimitsWithCorrectExecution(List, FDBRecordContext, RecordQueryPlan)}
* are far too strong for plans like this. Instead, we make very weak assertions that the byte scan limit does
* <em>something</em>.
*/
@ParameterizedTest
@MethodSource("complexTextQueries")
public void queryWithWideOrOfFullTextPrefixPredicates(@Nonnull RecordQuery query, int numPredicates) throws Exception {
deleteSimpleRecords();
final List<String> textSamples = ImmutableList.of(TextSamples.ANGSTROM, TextSamples.ROMEO_AND_JULIET_PROLOGUE, TextSamples.AETHELRED, TextSamples.FRENCH, TextSamples.KOREAN);
RecordMetaDataHook indexHook = metaDataBuilder -> metaDataBuilder.addIndex(metaDataBuilder.getRecordType(SIMPLE_DOC), SIMPLE_TEXT_PREFIX);
try (FDBRecordContext context = openContext()) {
openTextRecordStore(context, indexHook);
for (int i = 0; i < textSamples.size(); i++) {
recordStore.saveRecord(TestRecordsTextProto.SimpleDocument.newBuilder().setDocId(i).setGroup(i % 2).setText(textSamples.get(i)).build());
}
commit(context);
}
setupPlanner(null);
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, descendant(unorderedUnion(Collections.nCopies(numPredicates, any(RecordQueryPlan.class)))));
long totalBytes;
Set<Long> noLimitRecordIds = new HashSet<>();
try (FDBRecordContext context = openContext()) {
openTextRecordStore(context, indexHook);
context.getTimer().reset();
RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, null, ExecuteProperties.SERIAL_EXECUTE);
RecordCursorResult<FDBQueriedRecord<Message>> result;
do {
result = cursor.onNext().get();
if (result.hasNext()) {
TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
record.mergeFrom(result.get().getRecord());
noLimitRecordIds.add(record.getDocId());
}
} while (result.hasNext());
totalBytes = byteCounter.getBytesScanned(context);
}
Set<Long> limitRecordIds = new HashSet<>();
try (FDBRecordContext context = openContext()) {
openTextRecordStore(context);
ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setScannedBytesLimit(0);
byte[] continuation = null;
do {
context.getTimer().reset();
RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(query, continuation, executeProperties.build());
RecordCursorResult<FDBQueriedRecord<Message>> result;
do {
result = cursor.onNext().get();
if (result.hasNext()) {
TestRecordsTextProto.SimpleDocument.Builder record = TestRecordsTextProto.SimpleDocument.newBuilder();
record.mergeFrom(result.get().getRecord());
limitRecordIds.add(record.getDocId());
}
} while (result.hasNext());
assertThat(byteCounter.getBytesScanned(context), lessThan(totalBytes));
continuation = result.getContinuation().toBytes();
if (continuation != null) {
assertEquals(RecordCursor.NoNextReason.BYTE_LIMIT_REACHED, result.getNoNextReason());
}
} while (continuation != null);
assertEquals(noLimitRecordIds, limitRecordIds);
}
}
use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreScanLimitTest method plansByContinuation.
@ParameterizedTest(name = "plansByContinuation() [{index}] {0}")
@MethodSource("plansWithoutFail")
public void plansByContinuation(String description, boolean fail, RecordQueryPlan plan) throws Exception {
int maximumToScan = getMaximumToScan(plan);
// include a scanLimit of 0, in which case all progress happens via the first "free" key-value scan.
for (int scanLimit = 0; scanLimit <= maximumToScan * 2; scanLimit = 2 * scanLimit + 1) {
final Function<FDBQueriedRecord<Message>, Long> getRecNo = r -> {
TestRecords1Proto.MySimpleRecord.Builder record = TestRecords1Proto.MySimpleRecord.newBuilder();
record.mergeFrom(r.getRecord());
return record.getRecNo();
};
final ExecuteProperties.Builder properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit);
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
final List<Long> allAtOnce;
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
allAtOnce = cursor.map(getRecNo).asList().get();
}
final List<Long> byContinuation = new ArrayList<>();
byte[] continuation = null;
do {
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, properties.build())) {
if (context.getTimer() != null) {
context.getTimer().reset();
}
RecordCursorResult<FDBQueriedRecord<Message>> result;
while ((result = cursor.getNext()).hasNext()) {
byContinuation.add(getRecNo.apply(result.get()));
}
continuation = result.getContinuation().toBytes();
int overrun = BaseCursorCountVisitor.getCount(cursor);
Optional<Integer> recordScanned = getRecordScanned(context);
if (recordScanned.isPresent()) {
assertThat(recordScanned.get(), lessThanOrEqualTo(Math.min(scanLimit + overrun, maximumToScan)));
}
}
} while (continuation != null);
assertEquals(allAtOnce, byContinuation);
}
}
}
use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.
the class FDBNestedFieldQueryTest method testNestedPrimaryKeyQuery.
/**
* Verify that record scans with nested primary keys works properly.
* Specifically, verify that a filter is implemented as a record scan in the case where there is a two-field
* primary key both of whose fields are nested in some header subrecord.
*/
@DualPlannerTest
public void testNestedPrimaryKeyQuery() throws Exception {
final RecordMetaDataHook hook = metaData -> {
metaData.getRecordType("MyRecord").setPrimaryKey(concat(field("header").nest(field("path")), field("header").nest(field("rec_no"))));
};
try (FDBRecordContext context = openContext()) {
openRecordWithHeader(context, hook);
saveHeaderRecord(1, "a", 0, "able");
saveHeaderRecord(2, "a", 3, "baker");
commit(context);
}
RecordQuery query = RecordQuery.newBuilder().setRecordType("MyRecord").setFilter(Query.and(Query.field("header").matches(Query.field("path").equalsValue("a")), Query.field("header").matches(Query.field("rec_no").equalsValue(2L)))).build();
// Scan([[a, 2],[a, 2]])
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, scan(bounds(hasTupleString("[[a, 2],[a, 2]]"))));
assertEquals(1265534819, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(136710600, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(-1817343447, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
try (FDBRecordContext context = openContext()) {
openRecordWithHeader(context, hook);
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan)) {
RecordCursorResult<FDBQueriedRecord<Message>> result = cursor.getNext();
assertTrue(result.hasNext());
TestRecordsWithHeaderProto.MyRecord record = parseMyRecord(result.get().getRecord());
assertEquals("baker", record.getStrValue());
assertFalse(cursor.getNext().hasNext());
}
TestHelpers.assertDiscardedNone(context);
}
}
use of com.apple.foundationdb.record.RecordCursorResult in project fdb-record-layer by FoundationDB.
the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.
@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
final int numberOfRecordsToSave = 2000;
final int numberOfResultsToReturn = 5;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
for (int i = 0; i < numberOfRecordsToSave; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
// Slightly larger prime.
recBuilder.setRecNo((2244 * i + 1649) % 2357);
recBuilder.setNumValue2(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
if (mode == SortWithoutIndexMode.DISALLOWED) {
assertThrows(RecordCoreException.class, () -> {
planner.plan(query);
});
return;
}
((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
// Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
// Skip + limit is enough to overflow memory buffer. Skips into middle of section.
final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
timer.reset();
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
while (true) {
RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
if (!next.hasNext()) {
break;
}
FDBQueriedRecord<Message> rec = next.get();
TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
myrec.mergeFrom(rec.getRecord());
assertTrue(sortedInts.hasNext());
assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
}
}
assertFalse(sortedInts.hasNext());
assertDiscardedNone(context);
int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
if (mode == SortWithoutIndexMode.MEMORY) {
assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
} else {
int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
}
}
}
Aggregations