use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreScanLimitTest method testSplitContinuation.
@Test
public void testSplitContinuation() throws Exception {
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, TEST_SPLIT_HOOK);
// Undo setupRecordStore().
recordStore.deleteAllRecords();
commit(context);
}
final String bigValue = Strings.repeat("X", SplitHelper.SPLIT_RECORD_SIZE + 10);
final String smallValue = Strings.repeat("Y", 5);
final List<FDBStoredRecord<Message>> createdRecords = new ArrayList<>();
createdRecords.add(saveAndSplitSimpleRecord(1L, smallValue, 1));
createdRecords.add(saveAndSplitSimpleRecord(2L, smallValue, 2));
createdRecords.add(saveAndSplitSimpleRecord(3L, bigValue, 3));
createdRecords.add(saveAndSplitSimpleRecord(4L, smallValue, 4));
createdRecords.add(saveAndSplitSimpleRecord(5L, bigValue, 5));
createdRecords.add(saveAndSplitSimpleRecord(6L, bigValue, 6));
createdRecords.add(saveAndSplitSimpleRecord(7L, smallValue, 7));
createdRecords.add(saveAndSplitSimpleRecord(8L, smallValue, 8));
createdRecords.add(saveAndSplitSimpleRecord(9L, smallValue, 9));
// Scan one record at a time using continuations
final List<FDBStoredRecord<Message>> scannedRecords = new ArrayList<>();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, TEST_SPLIT_HOOK);
Supplier<ScanProperties> props = () -> new ScanProperties(ExecuteProperties.newBuilder().setScannedRecordsLimit(0).setIsolationLevel(IsolationLevel.SERIALIZABLE).build());
RecordCursorIterator<FDBStoredRecord<Message>> messageCursor = recordStore.scanRecords(null, props.get()).asIterator();
while (messageCursor.hasNext()) {
scannedRecords.add(messageCursor.next());
messageCursor = recordStore.scanRecords(messageCursor.getContinuation(), props.get()).asIterator();
}
commit(context);
}
assertEquals(createdRecords, scannedRecords);
}
use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.
the class KeySpaceDirectoryTest method testListAcrossTransactions.
@Test
public void testListAcrossTransactions() throws Exception {
KeySpace root = new KeySpace(new KeySpaceDirectory("a", KeyType.LONG, random.nextLong()).addSubdirectory(new KeySpaceDirectory("b", KeyType.STRING)));
final FDBDatabase database = FDBDatabaseFactory.instance().getDatabase();
final List<String> directoryEntries = IntStream.range(0, 10).boxed().map(i -> "val_" + i).collect(Collectors.toList());
final KeySpacePath rootPath = root.path("a");
try (final FDBRecordContext context = database.openContext()) {
final Transaction tr = context.ensureActive();
directoryEntries.forEach(name -> tr.set(rootPath.add("b", name).toTuple(context).pack(), TupleHelpers.EMPTY.pack()));
context.commit();
}
byte[] continuation = null;
int idx = 0;
do {
try (final FDBRecordContext context = database.openContext()) {
final RecordCursor<ResolvedKeySpacePath> cursor = rootPath.listSubdirectoryAsync(context, "b", continuation, new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build()));
List<ResolvedKeySpacePath> subdirs = context.asyncToSync(FDBStoreTimer.Waits.WAIT_KEYSPACE_LIST, cursor.asList());
if (!subdirs.isEmpty()) {
assertEquals(2, subdirs.size(), "Wrong number of path entries returned");
assertEquals("val_" + idx, subdirs.get(0).getResolvedValue());
assertEquals("val_" + (idx + 1), subdirs.get(1).getResolvedValue());
idx += 2;
continuation = cursor.getNext().getContinuation().toBytes();
System.out.println(continuation == null ? "null" : Tuple.fromBytes(continuation));
} else {
continuation = cursor.getNext().getContinuation().toBytes();
assertNull(continuation);
}
}
} while (continuation != null);
assertEquals(directoryEntries.size(), idx);
}
use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStoreByteLimitTest method testSplitContinuation.
@Test
public void testSplitContinuation() throws Exception {
deleteSimpleRecords();
final String bigValue = Strings.repeat("X", SplitHelper.SPLIT_RECORD_SIZE + 10);
final String smallValue = Strings.repeat("Y", 5);
final List<FDBStoredRecord<Message>> createdRecords = new ArrayList<>();
createdRecords.add(saveAndSplitSimpleRecord(1L, smallValue, 1));
createdRecords.add(saveAndSplitSimpleRecord(2L, smallValue, 2));
createdRecords.add(saveAndSplitSimpleRecord(3L, bigValue, 3));
createdRecords.add(saveAndSplitSimpleRecord(4L, smallValue, 4));
createdRecords.add(saveAndSplitSimpleRecord(5L, bigValue, 5));
createdRecords.add(saveAndSplitSimpleRecord(6L, bigValue, 6));
createdRecords.add(saveAndSplitSimpleRecord(7L, smallValue, 7));
createdRecords.add(saveAndSplitSimpleRecord(8L, smallValue, 8));
createdRecords.add(saveAndSplitSimpleRecord(9L, smallValue, 9));
// Scan one record at a time using continuations
final List<FDBStoredRecord<Message>> scannedRecords = new ArrayList<>();
try (FDBRecordContext context = openContext()) {
openSimpleRecordStore(context, TEST_SPLIT_HOOK);
Supplier<ScanProperties> props = () -> new ScanProperties(ExecuteProperties.newBuilder().setScannedBytesLimit(0).setIsolationLevel(IsolationLevel.SERIALIZABLE).build());
RecordCursorIterator<FDBStoredRecord<Message>> messageCursor = recordStore.scanRecords(null, props.get()).asIterator();
while (messageCursor.hasNext()) {
scannedRecords.add(messageCursor.next());
messageCursor = recordStore.scanRecords(messageCursor.getContinuation(), props.get()).asIterator();
}
commit(context);
}
assertEquals(createdRecords, scannedRecords);
}
use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method getRecordCountForRebuildIndexes.
/**
* Get count of records to pass to a {@link UserVersionChecker} to decide whether to build right away. If all of the
* new indexes are over a single type and that type has a record key prefix, then this count will only be over the
* record type being indexed. If not, it will be the count of all records of all types, as in that case, the indexer
* will need to scan the entire store to build each index. If determining the record count would be too costly (such
* as if there is not an appropriate {@linkplain IndexTypes#COUNT count} index defined), this function may return
* {@link Long#MAX_VALUE} to indicate that an unknown and unbounded number of records would have to be scanned
* to build the index.
*
* @param newStore {@code true} if this is a brand new store
* @param rebuildRecordCounts {@code true} if there is a record count key that needs to be rebuilt
* @param indexes indexes that need to be built
* @param singleRecordTypeWithPrefixKey either a single record type prefixed by the record type key or {@code null}
* @return a future that completes to the record count for the version checker
*/
@Nonnull
@SuppressWarnings("PMD.EmptyCatchBlock")
protected CompletableFuture<Long> getRecordCountForRebuildIndexes(boolean newStore, boolean rebuildRecordCounts, @Nonnull Map<Index, List<RecordType>> indexes, @Nullable RecordType singleRecordTypeWithPrefixKey) {
// Do this with the new indexes in write-only mode to avoid using one of them
// when evaluating the snapshot record count.
MutableRecordStoreState writeOnlyState = recordStoreStateRef.get().withWriteOnlyIndexes(indexes.keySet().stream().map(Index::getName).collect(Collectors.toList()));
if (singleRecordTypeWithPrefixKey != null) {
// Get a count for just those records, either from a COUNT index on just that type or from a universal COUNT index grouped by record type.
MutableRecordStoreState saveState = recordStoreStateRef.get();
try {
recordStoreStateRef.set(writeOnlyState);
return getSnapshotRecordCountForRecordType(singleRecordTypeWithPrefixKey.getName());
} catch (RecordCoreException ex) {
// No such index; have to use total record count.
} finally {
recordStoreStateRef.set(saveState);
}
}
if (!rebuildRecordCounts) {
MutableRecordStoreState saveState = recordStoreStateRef.get();
try {
recordStoreStateRef.set(writeOnlyState);
// See: FDBRecordStoreBase.checkPossiblyRebuild() could take a long time if the record count index is split into many groups (https://github.com/FoundationDB/fdb-record-layer/issues/7)
return getSnapshotRecordCount();
} catch (RecordCoreException ex) {
// Probably this was from the lack of appropriate index on count; treat like rebuildRecordCounts = true.
} finally {
recordStoreStateRef.set(saveState);
}
}
// Do a scan (limited to a single record) to see if the store is empty.
final ExecuteProperties executeProperties = ExecuteProperties.newBuilder().setReturnedRowLimit(1).setIsolationLevel(IsolationLevel.SNAPSHOT).build();
final ScanProperties scanProperties = new ScanProperties(executeProperties);
final RecordCursor<FDBStoredRecord<Message>> records;
if (singleRecordTypeWithPrefixKey == null) {
records = scanRecords(null, scanProperties);
} else {
records = scanRecords(TupleRange.allOf(singleRecordTypeWithPrefixKey.getRecordTypeKeyTuple()), null, scanProperties);
}
return records.onNext().thenApply(result -> {
if (result.hasNext()) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info(KeyValueLogMessage.of("version check scan found non-empty store", subspaceProvider.logKey(), subspaceProvider.toString(context)));
}
return Long.MAX_VALUE;
} else {
if (newStore ? LOGGER.isDebugEnabled() : LOGGER.isInfoEnabled()) {
KeyValueLogMessage msg = KeyValueLogMessage.build("version check scan found empty store", subspaceProvider.logKey(), subspaceProvider.toString(context));
if (newStore) {
LOGGER.debug(msg.toString());
} else {
LOGGER.info(msg.toString());
}
}
return 0L;
}
});
}
use of com.apple.foundationdb.record.ScanProperties in project fdb-record-layer by FoundationDB.
the class FDBRecordStore method scanTypedRecords.
@Nonnull
public <M extends Message> RecordCursor<FDBStoredRecord<M>> scanTypedRecords(@Nonnull RecordSerializer<M> typedSerializer, @Nullable final Tuple low, @Nullable final Tuple high, @Nonnull final EndpointType lowEndpoint, @Nonnull final EndpointType highEndpoint, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
final RecordMetaData metaData = metaDataProvider.getRecordMetaData();
final Subspace recordsSubspace = recordsSubspace();
final SplitHelper.SizeInfo sizeInfo = new SplitHelper.SizeInfo();
final RecordCursor<FDBRawRecord> rawRecords;
if (metaData.isSplitLongRecords()) {
RecordCursor<KeyValue> keyValues = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint).setScanProperties(scanProperties.with(ExecuteProperties::clearRowAndTimeLimits).with(ExecuteProperties::clearState)).build();
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValues, useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
} else {
KeyValueCursor.Builder keyValuesBuilder = KeyValueCursor.Builder.withSubspace(recordsSubspace).setContext(context).setContinuation(continuation).setLow(low, lowEndpoint).setHigh(high, highEndpoint);
if (omitUnsplitRecordSuffix) {
rawRecords = keyValuesBuilder.setScanProperties(scanProperties).build().map(kv -> {
sizeInfo.set(kv);
Tuple primaryKey = SplitHelper.unpackKey(recordsSubspace, kv);
return new FDBRawRecord(primaryKey, kv.getValue(), null, sizeInfo);
});
} else {
final ScanProperties finalScanProperties = scanProperties.with(executeProperties -> {
final ExecuteProperties.Builder builder = executeProperties.toBuilder().clearTimeLimit().clearSkipAndAdjustLimit().clearState();
int returnedRowLimit = builder.getReturnedRowLimitOrMax();
if (returnedRowLimit != Integer.MAX_VALUE) {
// Adjust limit to twice the supplied limit in case there are versions in the records
builder.setReturnedRowLimit(2 * returnedRowLimit);
}
return builder.build();
});
rawRecords = new SplitHelper.KeyValueUnsplitter(context, recordsSubspace, keyValuesBuilder.setScanProperties(finalScanProperties).build(), useOldVersionFormat(), sizeInfo, scanProperties.isReverse(), new CursorLimitManager(context, scanProperties.with(ExecuteProperties::clearReturnedRowLimit))).skip(scanProperties.getExecuteProperties().getSkip()).limitRowsTo(scanProperties.getExecuteProperties().getReturnedRowLimit());
}
}
RecordCursor<FDBStoredRecord<M>> result = rawRecords.mapPipelined(rawRecord -> {
final Optional<CompletableFuture<FDBRecordVersion>> versionFutureOptional;
if (useOldVersionFormat()) {
// Older format versions: do a separate read to get the version.
versionFutureOptional = loadRecordVersionAsync(rawRecord.getPrimaryKey(), scanProperties.getExecuteProperties().getIsolationLevel().isSnapshot());
} else {
// Newer format versions: the version is either in the record or it is not -- do not do another read.
versionFutureOptional = Optional.empty();
}
return deserializeRecord(typedSerializer, rawRecord, metaData, versionFutureOptional);
}, pipelineSizer.getPipelineSize(PipelineOperation.KEY_TO_RECORD));
return context.instrument(FDBStoreTimer.Events.SCAN_RECORDS, result);
}
Aggregations