Search in sources :

Example 1 with TransactionContext

use of com.apple.foundationdb.TransactionContext in project fdb-record-layer by FoundationDB.

the class BunchedMap method compact.

/**
 * Compact the values within the map into as few keys as possible. This will scan through and re-write
 * the keys to be optimal. This feature is experimental at the moment, but it should be used to better
 * pack entries if needed.
 *
 * @param tcx database or transaction to use when compacting data
 * @param subspace subspace within which the map's data are located
 * @param keyLimit maximum number of database keys to read in a single transaction
 * @param continuation the continuation returned from a previous call or <code>null</code>
 *                     to start from the beginning of the subspace
 * @return future that will complete with a continuation that can be used to complete
 *         the compaction across multiple transactions (<code>null</code> if finished)
 */
@Nonnull
protected CompletableFuture<byte[]> compact(@Nonnull TransactionContext tcx, @Nonnull Subspace subspace, int keyLimit, @Nullable byte[] continuation) {
    return tcx.runAsync(tr -> {
        byte[] subspaceKey = subspace.getKey();
        byte[] begin = (continuation == null) ? subspaceKey : continuation;
        byte[] end = subspace.range().end;
        final AsyncIterable<KeyValue> iterable = tr.snapshot().getRange(begin, end, keyLimit);
        List<Map.Entry<K, V>> currentEntryList = new ArrayList<>(bunchSize);
        // The estimated size can be off (and will be off for many implementations of BunchedSerializer),
        // but it is just a heuristic to know when to split, so that's fine (I claim).
        AtomicInteger currentEntrySize = new AtomicInteger(0);
        AtomicInteger readKeys = new AtomicInteger(0);
        AtomicReference<byte[]> lastReadKeyBytes = new AtomicReference<>(null);
        AtomicReference<K> lastKey = new AtomicReference<>(null);
        return AsyncUtil.forEach(iterable, kv -> {
            final K boundaryKey = serializer.deserializeKey(kv.getKey(), subspaceKey.length);
            final List<Map.Entry<K, V>> entriesFromKey = serializer.deserializeEntries(boundaryKey, kv.getValue());
            readKeys.incrementAndGet();
            if (entriesFromKey.size() >= bunchSize && currentEntryList.isEmpty()) {
                // Nothing can be done. Just move on.
                lastReadKeyBytes.set(null);
                return;
            }
            if (lastReadKeyBytes.get() == null) {
                lastReadKeyBytes.set(kv.getKey());
            }
            final byte[] endKeyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(entriesFromKey.get(entriesFromKey.size() - 1).getKey()), ZERO_ARRAY);
            tr.addReadConflictRange(lastReadKeyBytes.get(), endKeyBytes);
            tr.addWriteConflictRange(lastReadKeyBytes.get(), kv.getKey());
            lastReadKeyBytes.set(endKeyBytes);
            tr.clear(kv.getKey());
            instrumentDelete(kv.getKey(), kv.getValue());
            for (Map.Entry<K, V> entry : entriesFromKey) {
                byte[] serializedEntry = serializer.serializeEntry(entry);
                if (currentEntrySize.get() + serializedEntry.length > MAX_VALUE_SIZE && !currentEntryList.isEmpty()) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
                currentEntryList.add(entry);
                currentEntrySize.addAndGet(serializedEntry.length);
                if (currentEntryList.size() == bunchSize) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
            }
        }, tr.getExecutor()).thenApply(vignore -> {
            if (!currentEntryList.isEmpty()) {
                flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
            }
            // Return a valid continuation if there might be more keys
            if (lastKey.get() != null && keyLimit != ReadTransaction.ROW_LIMIT_UNLIMITED && readKeys.get() == keyLimit) {
                return ByteArrayUtil.join(subspaceKey, serializer.serializeKey(lastKey.get()), ZERO_ARRAY);
            } else {
                return null;
            }
        });
    });
}
Also used : AsyncPeekCallbackIterator(com.apple.foundationdb.async.AsyncPeekCallbackIterator) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.util.LogMessageKeys) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) MutationType(com.apple.foundationdb.MutationType) Transaction(com.apple.foundationdb.Transaction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) StreamingMode(com.apple.foundationdb.StreamingMode) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ReadTransaction(com.apple.foundationdb.ReadTransaction) TransactionContext(com.apple.foundationdb.TransactionContext) Consumer(java.util.function.Consumer) AbstractMap(java.util.AbstractMap) List(java.util.List) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) KeySelector(com.apple.foundationdb.KeySelector) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) Comparator(java.util.Comparator) Collections(java.util.Collections) AsyncPeekIterator(com.apple.foundationdb.async.AsyncPeekIterator) KeyValue(com.apple.foundationdb.KeyValue) ArrayList(java.util.ArrayList) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) AbstractMap(java.util.AbstractMap) Nonnull(javax.annotation.Nonnull)

Aggregations

KeySelector (com.apple.foundationdb.KeySelector)1 KeyValue (com.apple.foundationdb.KeyValue)1 MutationType (com.apple.foundationdb.MutationType)1 ReadTransaction (com.apple.foundationdb.ReadTransaction)1 StreamingMode (com.apple.foundationdb.StreamingMode)1 Transaction (com.apple.foundationdb.Transaction)1 TransactionContext (com.apple.foundationdb.TransactionContext)1 API (com.apple.foundationdb.annotation.API)1 AsyncIterable (com.apple.foundationdb.async.AsyncIterable)1 AsyncPeekCallbackIterator (com.apple.foundationdb.async.AsyncPeekCallbackIterator)1 AsyncPeekIterator (com.apple.foundationdb.async.AsyncPeekIterator)1 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)1 Subspace (com.apple.foundationdb.subspace.Subspace)1 ByteArrayUtil (com.apple.foundationdb.tuple.ByteArrayUtil)1 ByteArrayUtil2 (com.apple.foundationdb.tuple.ByteArrayUtil2)1 LogMessageKeys (com.apple.foundationdb.util.LogMessageKeys)1 AbstractMap (java.util.AbstractMap)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1