use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.
the class TimeWindowLeaderboardIndexMaintainer method deleteWhere.
@Override
public CompletableFuture<Void> deleteWhere(Transaction tr, @Nonnull Tuple prefix) {
return loadDirectory().thenApply(directory -> {
if (directory != null) {
final Subspace indexSubspace = getIndexSubspace();
final Subspace extraSubspace = getSecondarySubspace();
for (Iterable<TimeWindowLeaderboard> directoryEntry : directory.getLeaderboards().values()) {
for (TimeWindowLeaderboard leaderboard : directoryEntry) {
// Range deletes are used for the more common operation of deleting a time window, so we need
// to do each one here to get to its grouping.
final Tuple leaderboardGroupKey = leaderboard.getSubspaceKey().addAll(prefix);
// NOTE: Range.startsWith(), Subspace.range() and so on cover keys *strictly* within the range, but we
// may store something at the group root as well.
final byte[] indexKey = indexSubspace.pack(leaderboardGroupKey);
tr.clear(indexKey, ByteArrayUtil.strinc(indexKey));
final byte[] ranksetKey = extraSubspace.pack(leaderboardGroupKey);
tr.clear(ranksetKey, ByteArrayUtil.strinc(ranksetKey));
}
}
}
return null;
});
}
use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.
the class KeySpaceDirectory method listSubdirectoryAsync.
@Nonnull
// SonarQube doesn't realize that the cursor is wrapped and returned
@SuppressWarnings("squid:S2095")
protected RecordCursor<ResolvedKeySpacePath> listSubdirectoryAsync(@Nullable KeySpacePath listFrom, @Nonnull FDBRecordContext context, @Nonnull String subdirName, @Nullable ValueRange<?> valueRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
if (listFrom != null && listFrom.getDirectory() != this) {
throw new RecordCoreException("Provided path does not belong to this directory").addLogInfo("path", listFrom, "directory", this.getName());
}
final KeySpaceDirectory subdir = getSubdirectory(subdirName);
final CompletableFuture<ResolvedKeySpacePath> resolvedFromFuture = listFrom == null ? CompletableFuture.completedFuture(null) : listFrom.toResolvedPathAsync(context);
// The chained cursor cannot implement reverse scan, so we implement it by having the
// inner key value cursor do the reversing but telling the chained cursor we are moving
// forward.
final ScanProperties chainedCursorScanProperties;
if (scanProperties.isReverse()) {
chainedCursorScanProperties = scanProperties.setReverse(false);
} else {
chainedCursorScanProperties = scanProperties;
}
// For the read of the individual row keys, we only want to read a single key. In addition,
// the ChainedCursor is going to do counting of our reads to apply any limits that were specified
// on the ScanProperties. We don't want the inner KeyValueCursor in nextTuple() to ALSO count those
// same reads so we clear out its limits.
final ScanProperties keyReadScanProperties = scanProperties.with(props -> props.clearState().setReturnedRowLimit(1));
return new LazyCursor<>(resolvedFromFuture.thenCompose(resolvedFrom -> {
final Subspace subspace = resolvedFrom == null ? new Subspace() : resolvedFrom.toSubspace();
return subdir.getValueRange(context, valueRange, subspace).thenApply(range -> {
final RecordCursor<Tuple> cursor = new ChainedCursor<>(context, lastKey -> nextTuple(context, subspace, range, lastKey, keyReadScanProperties), Tuple::pack, Tuple::fromBytes, continuation, chainedCursorScanProperties);
return cursor.mapPipelined(tuple -> {
final Tuple key = Tuple.fromList(tuple.getItems());
return findChildForKey(context, resolvedFrom, key, 1, 0);
}, 1);
});
}), context.getExecutor());
}
use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.
the class BunchedMap method compact.
/**
* Compact the values within the map into as few keys as possible. This will scan through and re-write
* the keys to be optimal. This feature is experimental at the moment, but it should be used to better
* pack entries if needed.
*
* @param tcx database or transaction to use when compacting data
* @param subspace subspace within which the map's data are located
* @param keyLimit maximum number of database keys to read in a single transaction
* @param continuation the continuation returned from a previous call or <code>null</code>
* to start from the beginning of the subspace
* @return future that will complete with a continuation that can be used to complete
* the compaction across multiple transactions (<code>null</code> if finished)
*/
@Nonnull
protected CompletableFuture<byte[]> compact(@Nonnull TransactionContext tcx, @Nonnull Subspace subspace, int keyLimit, @Nullable byte[] continuation) {
return tcx.runAsync(tr -> {
byte[] subspaceKey = subspace.getKey();
byte[] begin = (continuation == null) ? subspaceKey : continuation;
byte[] end = subspace.range().end;
final AsyncIterable<KeyValue> iterable = tr.snapshot().getRange(begin, end, keyLimit);
List<Map.Entry<K, V>> currentEntryList = new ArrayList<>(bunchSize);
// The estimated size can be off (and will be off for many implementations of BunchedSerializer),
// but it is just a heuristic to know when to split, so that's fine (I claim).
AtomicInteger currentEntrySize = new AtomicInteger(0);
AtomicInteger readKeys = new AtomicInteger(0);
AtomicReference<byte[]> lastReadKeyBytes = new AtomicReference<>(null);
AtomicReference<K> lastKey = new AtomicReference<>(null);
return AsyncUtil.forEach(iterable, kv -> {
final K boundaryKey = serializer.deserializeKey(kv.getKey(), subspaceKey.length);
final List<Map.Entry<K, V>> entriesFromKey = serializer.deserializeEntries(boundaryKey, kv.getValue());
readKeys.incrementAndGet();
if (entriesFromKey.size() >= bunchSize && currentEntryList.isEmpty()) {
// Nothing can be done. Just move on.
lastReadKeyBytes.set(null);
return;
}
if (lastReadKeyBytes.get() == null) {
lastReadKeyBytes.set(kv.getKey());
}
final byte[] endKeyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(entriesFromKey.get(entriesFromKey.size() - 1).getKey()), ZERO_ARRAY);
tr.addReadConflictRange(lastReadKeyBytes.get(), endKeyBytes);
tr.addWriteConflictRange(lastReadKeyBytes.get(), kv.getKey());
lastReadKeyBytes.set(endKeyBytes);
tr.clear(kv.getKey());
instrumentDelete(kv.getKey(), kv.getValue());
for (Map.Entry<K, V> entry : entriesFromKey) {
byte[] serializedEntry = serializer.serializeEntry(entry);
if (currentEntrySize.get() + serializedEntry.length > MAX_VALUE_SIZE && !currentEntryList.isEmpty()) {
flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
currentEntrySize.set(0);
}
currentEntryList.add(entry);
currentEntrySize.addAndGet(serializedEntry.length);
if (currentEntryList.size() == bunchSize) {
flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
currentEntrySize.set(0);
}
}
}, tr.getExecutor()).thenApply(vignore -> {
if (!currentEntryList.isEmpty()) {
flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
}
// Return a valid continuation if there might be more keys
if (lastKey.get() != null && keyLimit != ReadTransaction.ROW_LIMIT_UNLIMITED && readKeys.get() == keyLimit) {
return ByteArrayUtil.join(subspaceKey, serializer.serializeKey(lastKey.get()), ZERO_ARRAY);
} else {
return null;
}
});
});
}
use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.
the class BunchedTupleSerializerTest method serializeKey.
@Test
public void serializeKey() {
List<Tuple> sortedTuples = TEST_TUPLES.stream().sorted().collect(Collectors.toList());
List<Tuple> sortedKeys = TEST_TUPLES.stream().map(serializer::serializeKey).sorted(ByteArrayUtil::compareUnsigned).map(serializer::deserializeKey).collect(Collectors.toList());
assertEquals(sortedTuples, sortedKeys);
// Add a subspace and make sure unpacking by length works.
Subspace subspace = new Subspace(Tuple.from("fake", "subspace"));
List<Tuple> prefixedTuples = TEST_TUPLES.stream().map(serializer::serializeKey).map(b -> ByteArrayUtil.join(subspace.getKey(), b)).map(data -> serializer.deserializeKey(data, subspace.getKey().length)).collect(Collectors.toList());
assertEquals(TEST_TUPLES, prefixedTuples);
}
use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.
the class TextIndexTest method scanMulti.
@Nonnull
private List<BunchedMapScanEntry<Tuple, List<Integer>, String>> scanMulti(@Nonnull FDBRecordStore store, @Nonnull Subspace mapSubspace) throws ExecutionException, InterruptedException {
SubspaceSplitter<String> splitter = new SubspaceSplitter<String>() {
@Nonnull
@Override
public Subspace subspaceOf(@Nonnull byte[] keyBytes) {
Tuple t = mapSubspace.unpack(keyBytes);
return mapSubspace.subspace(TupleHelpers.subTuple(t, 0, 1));
}
@Nonnull
@Override
public String subspaceTag(@Nonnull Subspace subspace) {
return mapSubspace.unpack(subspace.getKey()).getString(0);
}
};
BunchedMapMultiIterator<Tuple, List<Integer>, String> iterator = BUNCHED_MAP.scanMulti(store.ensureContextActive(), mapSubspace, splitter);
return AsyncUtil.collectRemaining(iterator).get();
}
Aggregations