Search in sources :

Example 21 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class TimeWindowLeaderboardIndexMaintainer method deleteWhere.

@Override
public CompletableFuture<Void> deleteWhere(Transaction tr, @Nonnull Tuple prefix) {
    return loadDirectory().thenApply(directory -> {
        if (directory != null) {
            final Subspace indexSubspace = getIndexSubspace();
            final Subspace extraSubspace = getSecondarySubspace();
            for (Iterable<TimeWindowLeaderboard> directoryEntry : directory.getLeaderboards().values()) {
                for (TimeWindowLeaderboard leaderboard : directoryEntry) {
                    // Range deletes are used for the more common operation of deleting a time window, so we need
                    // to do each one here to get to its grouping.
                    final Tuple leaderboardGroupKey = leaderboard.getSubspaceKey().addAll(prefix);
                    // NOTE: Range.startsWith(), Subspace.range() and so on cover keys *strictly* within the range, but we
                    // may store something at the group root as well.
                    final byte[] indexKey = indexSubspace.pack(leaderboardGroupKey);
                    tr.clear(indexKey, ByteArrayUtil.strinc(indexKey));
                    final byte[] ranksetKey = extraSubspace.pack(leaderboardGroupKey);
                    tr.clear(ranksetKey, ByteArrayUtil.strinc(ranksetKey));
                }
            }
        }
        return null;
    });
}
Also used : Subspace(com.apple.foundationdb.subspace.Subspace) Tuple(com.apple.foundationdb.tuple.Tuple)

Example 22 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class KeySpaceDirectory method listSubdirectoryAsync.

@Nonnull
// SonarQube doesn't realize that the cursor is wrapped and returned
@SuppressWarnings("squid:S2095")
protected RecordCursor<ResolvedKeySpacePath> listSubdirectoryAsync(@Nullable KeySpacePath listFrom, @Nonnull FDBRecordContext context, @Nonnull String subdirName, @Nullable ValueRange<?> valueRange, @Nullable byte[] continuation, @Nonnull ScanProperties scanProperties) {
    if (listFrom != null && listFrom.getDirectory() != this) {
        throw new RecordCoreException("Provided path does not belong to this directory").addLogInfo("path", listFrom, "directory", this.getName());
    }
    final KeySpaceDirectory subdir = getSubdirectory(subdirName);
    final CompletableFuture<ResolvedKeySpacePath> resolvedFromFuture = listFrom == null ? CompletableFuture.completedFuture(null) : listFrom.toResolvedPathAsync(context);
    // The chained cursor cannot implement reverse scan, so we implement it by having the
    // inner key value cursor do the reversing but telling the chained cursor we are moving
    // forward.
    final ScanProperties chainedCursorScanProperties;
    if (scanProperties.isReverse()) {
        chainedCursorScanProperties = scanProperties.setReverse(false);
    } else {
        chainedCursorScanProperties = scanProperties;
    }
    // For the read of the individual row keys, we only want to read a single key. In addition,
    // the ChainedCursor is going to do counting of our reads to apply any limits that were specified
    // on the ScanProperties.  We don't want the inner KeyValueCursor in nextTuple() to ALSO count those
    // same reads so we clear out its limits.
    final ScanProperties keyReadScanProperties = scanProperties.with(props -> props.clearState().setReturnedRowLimit(1));
    return new LazyCursor<>(resolvedFromFuture.thenCompose(resolvedFrom -> {
        final Subspace subspace = resolvedFrom == null ? new Subspace() : resolvedFrom.toSubspace();
        return subdir.getValueRange(context, valueRange, subspace).thenApply(range -> {
            final RecordCursor<Tuple> cursor = new ChainedCursor<>(context, lastKey -> nextTuple(context, subspace, range, lastKey, keyReadScanProperties), Tuple::pack, Tuple::fromBytes, continuation, chainedCursorScanProperties);
            return cursor.mapPipelined(tuple -> {
                final Tuple key = Tuple.fromList(tuple.getItems());
                return findChildForKey(context, resolvedFrom, key, 1, 0);
            }, 1);
        });
    }), context.getExecutor());
}
Also used : Arrays(java.util.Arrays) KeyRange(com.apple.foundationdb.record.KeyRange) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) FDBRecordContext(com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) Tuple(com.apple.foundationdb.tuple.Tuple) EndpointType(com.apple.foundationdb.record.EndpointType) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) ScanProperties(com.apple.foundationdb.record.ScanProperties) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) FDBStoreTimer(com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer) ChainedCursor(com.apple.foundationdb.record.cursors.ChainedCursor) StringWriter(java.io.StringWriter) IOException(java.io.IOException) UUID(java.util.UUID) RecordCoreArgumentException(com.apple.foundationdb.record.RecordCoreArgumentException) Objects(java.util.Objects) KeyValueCursor(com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor) List(java.util.List) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Writer(java.io.Writer) RecordCursor(com.apple.foundationdb.record.RecordCursor) ValueRange(com.apple.foundationdb.record.ValueRange) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) BitSet(java.util.BitSet) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) LazyCursor(com.apple.foundationdb.record.cursors.LazyCursor) RecordCursor(com.apple.foundationdb.record.RecordCursor) ScanProperties(com.apple.foundationdb.record.ScanProperties) Subspace(com.apple.foundationdb.subspace.Subspace) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Example 23 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class BunchedMap method compact.

/**
 * Compact the values within the map into as few keys as possible. This will scan through and re-write
 * the keys to be optimal. This feature is experimental at the moment, but it should be used to better
 * pack entries if needed.
 *
 * @param tcx database or transaction to use when compacting data
 * @param subspace subspace within which the map's data are located
 * @param keyLimit maximum number of database keys to read in a single transaction
 * @param continuation the continuation returned from a previous call or <code>null</code>
 *                     to start from the beginning of the subspace
 * @return future that will complete with a continuation that can be used to complete
 *         the compaction across multiple transactions (<code>null</code> if finished)
 */
@Nonnull
protected CompletableFuture<byte[]> compact(@Nonnull TransactionContext tcx, @Nonnull Subspace subspace, int keyLimit, @Nullable byte[] continuation) {
    return tcx.runAsync(tr -> {
        byte[] subspaceKey = subspace.getKey();
        byte[] begin = (continuation == null) ? subspaceKey : continuation;
        byte[] end = subspace.range().end;
        final AsyncIterable<KeyValue> iterable = tr.snapshot().getRange(begin, end, keyLimit);
        List<Map.Entry<K, V>> currentEntryList = new ArrayList<>(bunchSize);
        // The estimated size can be off (and will be off for many implementations of BunchedSerializer),
        // but it is just a heuristic to know when to split, so that's fine (I claim).
        AtomicInteger currentEntrySize = new AtomicInteger(0);
        AtomicInteger readKeys = new AtomicInteger(0);
        AtomicReference<byte[]> lastReadKeyBytes = new AtomicReference<>(null);
        AtomicReference<K> lastKey = new AtomicReference<>(null);
        return AsyncUtil.forEach(iterable, kv -> {
            final K boundaryKey = serializer.deserializeKey(kv.getKey(), subspaceKey.length);
            final List<Map.Entry<K, V>> entriesFromKey = serializer.deserializeEntries(boundaryKey, kv.getValue());
            readKeys.incrementAndGet();
            if (entriesFromKey.size() >= bunchSize && currentEntryList.isEmpty()) {
                // Nothing can be done. Just move on.
                lastReadKeyBytes.set(null);
                return;
            }
            if (lastReadKeyBytes.get() == null) {
                lastReadKeyBytes.set(kv.getKey());
            }
            final byte[] endKeyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(entriesFromKey.get(entriesFromKey.size() - 1).getKey()), ZERO_ARRAY);
            tr.addReadConflictRange(lastReadKeyBytes.get(), endKeyBytes);
            tr.addWriteConflictRange(lastReadKeyBytes.get(), kv.getKey());
            lastReadKeyBytes.set(endKeyBytes);
            tr.clear(kv.getKey());
            instrumentDelete(kv.getKey(), kv.getValue());
            for (Map.Entry<K, V> entry : entriesFromKey) {
                byte[] serializedEntry = serializer.serializeEntry(entry);
                if (currentEntrySize.get() + serializedEntry.length > MAX_VALUE_SIZE && !currentEntryList.isEmpty()) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
                currentEntryList.add(entry);
                currentEntrySize.addAndGet(serializedEntry.length);
                if (currentEntryList.size() == bunchSize) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
            }
        }, tr.getExecutor()).thenApply(vignore -> {
            if (!currentEntryList.isEmpty()) {
                flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
            }
            // Return a valid continuation if there might be more keys
            if (lastKey.get() != null && keyLimit != ReadTransaction.ROW_LIMIT_UNLIMITED && readKeys.get() == keyLimit) {
                return ByteArrayUtil.join(subspaceKey, serializer.serializeKey(lastKey.get()), ZERO_ARRAY);
            } else {
                return null;
            }
        });
    });
}
Also used : AsyncPeekCallbackIterator(com.apple.foundationdb.async.AsyncPeekCallbackIterator) Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.util.LogMessageKeys) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) MutationType(com.apple.foundationdb.MutationType) Transaction(com.apple.foundationdb.Transaction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Nonnull(javax.annotation.Nonnull) StreamingMode(com.apple.foundationdb.StreamingMode) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) KeyValue(com.apple.foundationdb.KeyValue) ReadTransaction(com.apple.foundationdb.ReadTransaction) TransactionContext(com.apple.foundationdb.TransactionContext) Consumer(java.util.function.Consumer) AbstractMap(java.util.AbstractMap) List(java.util.List) AsyncIterable(com.apple.foundationdb.async.AsyncIterable) KeySelector(com.apple.foundationdb.KeySelector) Optional(java.util.Optional) API(com.apple.foundationdb.annotation.API) Comparator(java.util.Comparator) Collections(java.util.Collections) AsyncPeekIterator(com.apple.foundationdb.async.AsyncPeekIterator) KeyValue(com.apple.foundationdb.KeyValue) ArrayList(java.util.ArrayList) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) AbstractMap(java.util.AbstractMap) Nonnull(javax.annotation.Nonnull)

Example 24 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class BunchedTupleSerializerTest method serializeKey.

@Test
public void serializeKey() {
    List<Tuple> sortedTuples = TEST_TUPLES.stream().sorted().collect(Collectors.toList());
    List<Tuple> sortedKeys = TEST_TUPLES.stream().map(serializer::serializeKey).sorted(ByteArrayUtil::compareUnsigned).map(serializer::deserializeKey).collect(Collectors.toList());
    assertEquals(sortedTuples, sortedKeys);
    // Add a subspace and make sure unpacking by length works.
    Subspace subspace = new Subspace(Tuple.from("fake", "subspace"));
    List<Tuple> prefixedTuples = TEST_TUPLES.stream().map(serializer::serializeKey).map(b -> ByteArrayUtil.join(subspace.getKey(), b)).map(data -> serializer.deserializeKey(data, subspace.getKey().length)).collect(Collectors.toList());
    assertEquals(TEST_TUPLES, prefixedTuples);
}
Also used : Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) Arrays(java.util.Arrays) Collectors(java.util.stream.Collectors) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) Test(org.junit.jupiter.api.Test) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) Tuple(com.apple.foundationdb.tuple.Tuple) AbstractMap(java.util.AbstractMap) List(java.util.List) Map(java.util.Map) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Collections(java.util.Collections) Subspace(com.apple.foundationdb.subspace.Subspace) Tuple(com.apple.foundationdb.tuple.Tuple) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) Test(org.junit.jupiter.api.Test)

Example 25 with Subspace

use of com.apple.foundationdb.subspace.Subspace in project fdb-record-layer by FoundationDB.

the class TextIndexTest method scanMulti.

@Nonnull
private List<BunchedMapScanEntry<Tuple, List<Integer>, String>> scanMulti(@Nonnull FDBRecordStore store, @Nonnull Subspace mapSubspace) throws ExecutionException, InterruptedException {
    SubspaceSplitter<String> splitter = new SubspaceSplitter<String>() {

        @Nonnull
        @Override
        public Subspace subspaceOf(@Nonnull byte[] keyBytes) {
            Tuple t = mapSubspace.unpack(keyBytes);
            return mapSubspace.subspace(TupleHelpers.subTuple(t, 0, 1));
        }

        @Nonnull
        @Override
        public String subspaceTag(@Nonnull Subspace subspace) {
            return mapSubspace.unpack(subspace.getKey()).getString(0);
        }
    };
    BunchedMapMultiIterator<Tuple, List<Integer>, String> iterator = BUNCHED_MAP.scanMulti(store.ensureContextActive(), mapSubspace, splitter);
    return AsyncUtil.collectRemaining(iterator).get();
}
Also used : Nonnull(javax.annotation.Nonnull) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) List(java.util.List) Matchers.containsString(org.hamcrest.Matchers.containsString) PlanMatchers.hasTupleString(com.apple.foundationdb.record.query.plan.match.PlanMatchers.hasTupleString) SubspaceSplitter(com.apple.foundationdb.map.SubspaceSplitter) Tuple(com.apple.foundationdb.tuple.Tuple) Nonnull(javax.annotation.Nonnull)

Aggregations

Subspace (com.apple.foundationdb.subspace.Subspace)61 Nonnull (javax.annotation.Nonnull)33 Tuple (com.apple.foundationdb.tuple.Tuple)28 List (java.util.List)23 ArrayList (java.util.ArrayList)21 CompletableFuture (java.util.concurrent.CompletableFuture)21 Nullable (javax.annotation.Nullable)20 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)19 KeyValue (com.apple.foundationdb.KeyValue)18 Map (java.util.Map)18 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)17 API (com.apple.foundationdb.annotation.API)16 ScanProperties (com.apple.foundationdb.record.ScanProperties)15 Collections (java.util.Collections)15 Test (org.junit.jupiter.api.Test)15 Transaction (com.apple.foundationdb.Transaction)14 RecordCursor (com.apple.foundationdb.record.RecordCursor)14 TupleRange (com.apple.foundationdb.record.TupleRange)14 LogMessageKeys (com.apple.foundationdb.record.logging.LogMessageKeys)14 Message (com.google.protobuf.Message)14