use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.
the class FDBOrQueryToUnionTest method testOrQuery5WithLimits.
@DualPlannerTest
@MethodSource("query5WithLimitsArgs")
@ParameterizedTest(name = "testOrQuery5WithLimits [limit = {0}, removesDuplicates = {1}]")
void testOrQuery5WithLimits(int limit, boolean removesDuplicates) throws Exception {
RecordMetaDataHook hook = complexQuerySetupHook();
complexQuerySetup(hook);
setDeferFetchAfterUnionAndIntersection(true);
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.or(Query.field("str_value_indexed").lessThan("m"), Query.field("num_value_3_indexed").greaterThan(3))).setRemoveDuplicates(removesDuplicates).build();
// Unordered(Index(MySimpleRecord$str_value_indexed ([null],[m])) ∪ Index(MySimpleRecord$num_value_3_indexed ([3],>))
RecordQueryPlan plan = planner.plan(query);
final BindingMatcher<RecordQueryUnorderedUnionPlan> unionPlanBindingMatcher = unorderedUnionPlan(coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$str_value_indexed")))), coveringIndexPlan().where(indexPlanOf(indexPlan().where(indexName("MySimpleRecord$num_value_3_indexed")))));
final BindingMatcher<? extends RecordQueryPlan> planMatcher;
if (removesDuplicates) {
planMatcher = fetchFromPartialRecordPlan(unorderedPrimaryKeyDistinctPlan(unionPlanBindingMatcher));
} else {
planMatcher = fetchFromPartialRecordPlan(unionPlanBindingMatcher);
}
assertMatchesExactly(plan, planMatcher);
if (planner instanceof RecordQueryPlanner) {
assertEquals(removesDuplicates ? 1898767693 : 1898767686, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(removesDuplicates ? -583062018 : 212117636, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(removesDuplicates ? 1864525478 : -1635262164, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
} else {
assertEquals(removesDuplicates ? 1898767693 : 1898767686, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(removesDuplicates ? -583062018 : 212117636, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(removesDuplicates ? 1864525478 : -1635262164, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
}
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, hook);
boolean done = false;
byte[] continuation = null;
Set<Tuple> uniqueKeys = new HashSet<>();
int itr = 0;
while (!done) {
ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(limit).build();
try (RecordCursorIterator<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, continuation, executeProperties).asIterator()) {
int i = 0;
Set<Tuple> keysThisIteration = new HashSet<>();
while (cursor.hasNext()) {
FDBQueriedRecord<Message> rec = cursor.next();
TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
myrec.mergeFrom(Objects.requireNonNull(rec).getRecord());
assertTrue(myrec.getStrValueIndexed().compareTo("m") < 0 || myrec.getNumValue3Indexed() > 3);
uniqueKeys.add(rec.getPrimaryKey());
if (removesDuplicates) {
assertThat(keysThisIteration.add(rec.getPrimaryKey()), is(true));
}
i++;
}
continuation = cursor.getContinuation();
done = cursor.getNoNextReason().isSourceExhausted();
if (!done) {
assertEquals(limit, i);
}
}
itr++;
assertThat("exceeded maximum iterations", itr, lessThan(500));
}
assertEquals(50 + 10, uniqueKeys.size());
}
}
use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreQueryTest method queryWithShortTimeLimit.
/**
* Verify that simple queries execute properly with short time limits.
*/
@DualPlannerTest
void queryWithShortTimeLimit() throws Exception {
setupSimpleRecordStore(null, (i, builder) -> {
builder.setRecNo(i);
builder.setNumValue3Indexed(i / 10);
});
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").equalsValue(5)).build();
RecordQueryPlan plan = planner.plan(query);
ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(1000).setTimeLimit(1).setIsolationLevel(IsolationLevel.SERIALIZABLE).build();
List<Long> list = new ArrayList<>();
byte[] continuation = null;
int count = 0;
do {
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, null);
long timeLeft = context.getTransactionCreateTime() + executeProperties.getTimeLimit() - System.currentTimeMillis();
if (timeLeft > 0) {
Thread.sleep(timeLeft);
}
count++;
try (RecordCursor<Long> cursor = recordStore.executeQuery(plan, continuation, executeProperties).map(record -> TestRecords1Proto.MySimpleRecord.newBuilder().mergeFrom(record.getRecord()).getRecNo())) {
cursor.forEach(list::add).join();
RecordCursorResult<Long> result = cursor.getNext();
continuation = result.getContinuation().toBytes();
if (continuation == null) {
assertEquals(RecordCursor.NoNextReason.SOURCE_EXHAUSTED, result.getNoNextReason());
} else {
assertEquals(RecordCursor.NoNextReason.TIME_LIMIT_REACHED, result.getNoNextReason());
}
}
}
} while (continuation != null);
assertEquals(LongStream.range(50, 60).mapToObj(Long::valueOf).collect(Collectors.toList()), list);
assertEquals(11, count);
}
use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.
the class FDBSortQueryIndexSelectionTest method sortWithoutIndex.
@EnumSource(SortWithoutIndexMode.class)
@ParameterizedTest(name = "sortWithoutIndex [mode = {0}]")
public void sortWithoutIndex(SortWithoutIndexMode mode) throws Exception {
final int numberOfRecordsToSave = 2000;
final int numberOfResultsToReturn = 5;
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
for (int i = 0; i < numberOfRecordsToSave; i++) {
TestRecords1Proto.MySimpleRecord.Builder recBuilder = TestRecords1Proto.MySimpleRecord.newBuilder();
// Slightly larger prime.
recBuilder.setRecNo((2244 * i + 1649) % 2357);
recBuilder.setNumValue2(i);
recBuilder.setNumValue3Indexed(i % 5);
recordStore.saveRecord(recBuilder.build());
}
commit(context);
}
RecordQuery query = RecordQuery.newBuilder().setRecordType("MySimpleRecord").setFilter(Query.field("num_value_3_indexed").greaterThanOrEquals(2)).setSort(field("num_value_2"), true).build();
if (mode == SortWithoutIndexMode.DISALLOWED) {
assertThrows(RecordCoreException.class, () -> {
planner.plan(query);
});
return;
}
((RecordQueryPlanner) planner).setConfiguration(((RecordQueryPlanner) planner).getConfiguration().asBuilder().setAllowNonIndexSort(true).build());
// Index(MySimpleRecord$num_value_3_indexed [[2],>) ORDER BY num_value_2 DESC
RecordQueryPlan plan = planner.plan(query);
assertThat(plan, sort(allOf(hasProperty("reverse", equalTo(true))), indexScan(allOf(indexName("MySimpleRecord$num_value_3_indexed"), bounds(hasTupleString("[[2],>"))))));
assertEquals(256365917, plan.planHash(PlanHashable.PlanHashKind.LEGACY));
assertEquals(172993081, plan.planHash(PlanHashable.PlanHashKind.FOR_CONTINUATION));
assertEquals(748321565, plan.planHash(PlanHashable.PlanHashKind.STRUCTURAL_WITHOUT_LITERALS));
// Skip + limit is enough to overflow memory buffer. Skips into middle of section.
final int skip = mode == SortWithoutIndexMode.FILE ? numberOfRecordsToSave / 2 + 1 : 0;
ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder().setSkip(skip).setReturnedRowLimit(numberOfResultsToReturn);
final PrimitiveIterator.OfInt sortedInts = IntStream.iterate(numberOfRecordsToSave - 1, i -> i - 1).filter(i -> i % 5 >= 2).skip(skip).limit(numberOfResultsToReturn).iterator();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context);
timer.reset();
try (RecordCursor<FDBQueriedRecord<Message>> cursor = recordStore.executeQuery(plan, null, executeProperties.build())) {
while (true) {
RecordCursorResult<FDBQueriedRecord<Message>> next = cursor.getNext();
if (!next.hasNext()) {
break;
}
FDBQueriedRecord<Message> rec = next.get();
TestRecords1Proto.MySimpleRecord.Builder myrec = TestRecords1Proto.MySimpleRecord.newBuilder();
myrec.mergeFrom(rec.getRecord());
assertTrue(sortedInts.hasNext());
assertEquals(sortedInts.nextInt(), myrec.getNumValue2());
}
}
assertFalse(sortedInts.hasNext());
assertDiscardedNone(context);
int countBeforeSorting = (numberOfRecordsToSave / 5) * 3;
if (mode == SortWithoutIndexMode.MEMORY) {
assertEquals(0, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.MEMORY_SORT_STORE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.MEMORY_SORT_LOAD_RECORD));
} else {
int nfiles = numberOfRecordsToSave / RecordQuerySortAdapter.DEFAULT_MAX_RECORD_COUNT_IN_MEMORY;
assertEquals(nfiles, timer.getCount(SortEvents.Events.FILE_SORT_OPEN_FILE));
assertEquals(nfiles - 1, timer.getCount(SortEvents.Events.FILE_SORT_MERGE_FILES));
assertEquals(countBeforeSorting, timer.getCount(SortEvents.Events.FILE_SORT_SAVE_RECORD));
assertEquals(numberOfResultsToReturn, timer.getCount(SortEvents.Events.FILE_SORT_LOAD_RECORD));
assertEquals(skip / RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_SECTION));
assertEquals(skip % RecordQuerySortAdapter.DEFAULT_RECORD_COUNT_PER_SECTION, timer.getCount(SortEvents.Events.FILE_SORT_SKIP_RECORD));
assertThat(timer.getCount(SortEvents.Counts.FILE_SORT_FILE_BYTES), allOf(greaterThan(1000), lessThan(100000)));
}
}
}
use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method getRecordCountForRebuildIndexes.
/**
* Get count of records to pass to a {@link UserVersionChecker} to decide whether to build right away. If all of the
* new indexes are over a single type and that type has a record key prefix, then this count will only be over the
* record type being indexed. If not, it will be the count of all records of all types, as in that case, the indexer
* will need to scan the entire store to build each index. If determining the record count would be too costly (such
* as if there is not an appropriate {@linkplain IndexTypes#COUNT count} index defined), this function may return
* {@link Long#MAX_VALUE} to indicate that an unknown and unbounded number of records would have to be scanned
* to build the index.
*
* @param newStore {@code true} if this is a brand new store
* @param rebuildRecordCounts {@code true} if there is a record count key that needs to be rebuilt
* @param indexes indexes that need to be built
* @param singleRecordTypeWithPrefixKey either a single record type prefixed by the record type key or {@code null}
* @return a future that completes to the record count for the version checker
*/
@Nonnull
@SuppressWarnings("PMD.EmptyCatchBlock")
protected CompletableFuture<Long> getRecordCountForRebuildIndexes(boolean newStore, boolean rebuildRecordCounts, @Nonnull Map<Index, List<RecordType>> indexes, @Nullable RecordType singleRecordTypeWithPrefixKey) {
// Do this with the new indexes in write-only mode to avoid using one of them
// when evaluating the snapshot record count.
MutableRecordStoreState writeOnlyState = recordStoreStateRef.get().withWriteOnlyIndexes(indexes.keySet().stream().map(Index::getName).collect(Collectors.toList()));
if (singleRecordTypeWithPrefixKey != null) {
// Get a count for just those records, either from a COUNT index on just that type or from a universal COUNT index grouped by record type.
MutableRecordStoreState saveState = recordStoreStateRef.get();
try {
recordStoreStateRef.set(writeOnlyState);
return getSnapshotRecordCountForRecordType(singleRecordTypeWithPrefixKey.getName());
} catch (RecordCoreException ex) {
// No such index; have to use total record count.
} finally {
recordStoreStateRef.set(saveState);
}
}
if (!rebuildRecordCounts) {
MutableRecordStoreState saveState = recordStoreStateRef.get();
try {
recordStoreStateRef.set(writeOnlyState);
// See: FDBRecordStoreBase.checkPossiblyRebuild() could take a long time if the record count index is split into many groups (https://github.com/FoundationDB/fdb-record-layer/issues/7)
return getSnapshotRecordCount();
} catch (RecordCoreException ex) {
// Probably this was from the lack of appropriate index on count; treat like rebuildRecordCounts = true.
} finally {
recordStoreStateRef.set(saveState);
}
}
// Do a scan (limited to a single record) to see if the store is empty.
final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
final ScanProperties scanProperties = new ScanProperties(executeProperties);
final RecordCursor<FDBStoredRecord<Message>> records;
if (singleRecordTypeWithPrefixKey == null) {
records = scanRecords(null, scanProperties);
} else {
records = scanRecords(TupleRange.allOf(singleRecordTypeWithPrefixKey.getRecordTypeKeyTuple()), null, scanProperties);
}
return records.onNext().thenApply(result -> {
if (result.hasNext()) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(KeyValueLogMessage.of("version check scan found non-empty store", subspaceProvider.logKey(), subspaceProvider.toString(context)));
}
return Long.MAX_VALUE;
} else {
if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
KeyValueLogMessage msg = KeyValueLogMessage.build("version check scan found empty store", subspaceProvider.logKey(), subspaceProvider.toString(context));
if (newStore) {
LOGGER.debug(msg.toString());
} else {
LOGGER.info(msg.toString());
}
}
return 0L;
}
});
}
use of com.apple.foundationdb.record.ExecuteProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method scanTypedRecords.
@Nonnull
public <M extends Message> RecordCursor<FDBStoredRecord<M>> scanTypedRecords(@Nonnull RecordSerializer<M> typedSerializer, @Nullable final Tuple low, @Nullable final Tuple high, @Nonnull final EndpointType lowEndpoint, @Nonnull final EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
final Subspace recordsSubspace = recordsSubspace();
final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
final RecordCursor<FDBRawRecord> rawRecords;
if (metaData.isSplitLongRecords()) {
RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
} else {
KeyValueCursor.Builder keyValuesBuilder = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint);
if (omitUnsplitRecordSuffix) {
rawRecords = keyValuesBuilder.setScanProperties(scanProperties).build().map(kv -> {
sizeInfo.set(kv);
Tuple primaryKey = SplitHelper.unpackKey(recordsSubspace, kv);
return new FDBRawRecord(primaryKey, kv.getValue(), null, sizeInfo);
});
} else {
final ScanProperties finalScanProperties = scanProperties.with(executeProperties -> {
final ExecuteProperties.Builder builder = executeProperties.toBuilder().clearTimeLimit().clearSkipAndAdjustLimit().clearState();
int returnedRowLimit = builder.getReturnedRowLimitOrMax();
if (returnedRowLimit != Integer.MAX_VALUE) {
// Adjust limit to twice the supplied limit in case there are versions in the records
builder.setReturnedRowLimit(2 * returnedRowLimit);
}
return builder.build();
});
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValuesBuilder.setScanProperties(finalScanProperties).build(), useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
}
}
RecordCursor<FDBStoredRecord<M>> result = rawRecords.mapPipelined(rawRecord -> {
final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
if (useOldVersionFormat()) {
// Older format versions: do a separate read to get the version.
versionFutureOptional = loadRecordVersionAsync(rawRecord.getPrimaryKey(), scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot());
} else {
// Newer format versions: the version is either in the record or it is not -- do not do another read.
versionFutureOptional = Optional.empty();
}
return deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
}, pipelineSizer.getPipelineSize(PipelineOperation.KEY_TO_RECORD));
return context.instrument(FDBStoreTimer.Events.SCAN_RECORDS, result);
}
Aggregations