Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 11 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method insertSingleKey.

@Test
public void insertSingleKey() {
    List<Tuple> testTuples = Stream.of(1066L, 1776L, 1415L, 800L).map(Tuple::from).collect(Collectors.toList());
    Tuple value = Tuple.from(1415L);
    db.run(tr -> {
        Tuple minSoFar = null;
        for (int i = 0; i < testTuples.size(); i++) {
            Tuple key = testTuples.get(i);
            minSoFar = (minSoFar == null || key.compareTo(minSoFar) < 0) ? key : minSoFar;
            map.put(tr, bmSubspace, key, value).join();
            for (int j = 0; j < testTuples.size(); j++) {
                assertEquals(j <= i, map.containsKey(tr, bmSubspace, testTuples.get(j)).join());
            }
            List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
            assertEquals(1, rangeKVs.size());
            assertArrayEquals(bmSubspace.pack(minSoFar), rangeKVs.get(0).getKey());
            List<Map.Entry<Tuple, Tuple>> entryList = testTuples.subList(0, i + 1).stream().sorted().map(t -> new AbstractMap.SimpleImmutableEntry<>(t, value)).collect(Collectors.toList());
            assertArrayEquals(serializer.serializeEntries(entryList), rangeKVs.get(0).getValue());
        }
        return null;
    });
}

Example 12 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class VersionFromTimestamp method versionFromTimestamp.

private static CompletableFuture<Long> versionFromTimestamp(@Nonnull ReadTransaction tr, @Nonnull Instant timestamp, boolean start) {
    final byte[] dateKey = ByteArrayUtil.join(SystemKeyspace.TIMEKEEPER_KEY_PREFIX, Tuple.from(timestamp.getEpochSecond()).pack());
    final KeySelector startKey;
    final KeySelector endKey;
    if (start) {
        startKey = KeySelector.firstGreaterThan(SystemKeyspace.TIMEKEEPER_KEY_PREFIX);
        endKey = KeySelector.firstGreaterThan(dateKey);
    } else {
        startKey = KeySelector.firstGreaterOrEqual(dateKey);
        endKey = KeySelector.firstGreaterOrEqual(ByteArrayUtil.strinc(SystemKeyspace.TIMEKEEPER_KEY_PREFIX));
    }
    final AsyncIterator<KeyValue> range = tr.getRange(startKey, endKey, 1, start).iterator();
    return range.onHasNext().thenApply(hasNext -> {
        if (hasNext) {
            return Tuple.fromBytes(range.next().getValue()).getLong(0);
        } else if (start) {
            return 0L;
        } else {
            return Long.MAX_VALUE;
        }
    });
}

Example 13 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMap method entryForKey.

private CompletableFuture<Optional<KeyValue>> entryForKey(@Nonnull Transaction tr, @Nonnull byte[] subspaceKey, @Nonnull K key) {
    byte[] keyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(key));
    tr.addReadConflictKey(keyBytes);
    // result in additional elements being returned.
    return instrumentRangeRead(tr.snapshot().getRange(KeySelector.lastLessOrEqual(keyBytes), KeySelector.firstGreaterThan(keyBytes), ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList()).thenApply(keyValues -> {
        if (keyValues.isEmpty()) {
            // There aren't any entries before this key in the database.
            return Optional.empty();
        } else {
            // The last (and probably only) result of the range read should be
            // the greatest key that is less than or equal to keyBytes.
            KeyValue kv = keyValues.get(keyValues.size() - 1);
            if (ByteArrayUtil.compareUnsigned(kv.getKey(), keyBytes) > 0) {
                throw new BunchedMapException("signpost key found for key is greater than original key").addLogInfo(LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspaceKey)).addLogInfo("key", ByteArrayUtil2.loggable(keyBytes)).addLogInfo("signpostKey", ByteArrayUtil2.loggable(kv.getKey()));
            }
            if (ByteArrayUtil.startsWith(kv.getKey(), subspaceKey)) {
                // The candidate key is in the correct subspace, so this is the signpost key for the given key
                return Optional.of(kv);
            } else {
                // vacuously the case if the map is empty).
                return Optional.empty();
            }
        }
    });
}

Example 14 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMap method compact.

/**
 * Compact the values within the map into as few keys as possible. This will scan through and re-write
 * the keys to be optimal. This feature is experimental at the moment, but it should be used to better
 * pack entries if needed.
 *
 * @param tcx database or transaction to use when compacting data
 * @param subspace subspace within which the map's data are located
 * @param keyLimit maximum number of database keys to read in a single transaction
 * @param continuation the continuation returned from a previous call or <code>null</code>
 *                     to start from the beginning of the subspace
 * @return future that will complete with a continuation that can be used to complete
 *         the compaction across multiple transactions (<code>null</code> if finished)
 */
@Nonnull
protected CompletableFuture<byte[]> compact(@Nonnull TransactionContext tcx, @Nonnull Subspace subspace, int keyLimit, @Nullable byte[] continuation) {
    return tcx.runAsync(tr -> {
        byte[] subspaceKey = subspace.getKey();
        byte[] begin = (continuation == null) ? subspaceKey : continuation;
        byte[] end = subspace.range().end;
        final AsyncIterable<KeyValue> iterable = tr.snapshot().getRange(begin, end, keyLimit);
        List<Map.Entry<K, V>> currentEntryList = new ArrayList<>(bunchSize);
        // The estimated size can be off (and will be off for many implementations of BunchedSerializer),
        // but it is just a heuristic to know when to split, so that's fine (I claim).
        AtomicInteger currentEntrySize = new AtomicInteger(0);
        AtomicInteger readKeys = new AtomicInteger(0);
        AtomicReference<byte[]> lastReadKeyBytes = new AtomicReference<>(null);
        AtomicReference<K> lastKey = new AtomicReference<>(null);
        return AsyncUtil.forEach(iterable, kv -> {
            final K boundaryKey = serializer.deserializeKey(kv.getKey(), subspaceKey.length);
            final List<Map.Entry<K, V>> entriesFromKey = serializer.deserializeEntries(boundaryKey, kv.getValue());
            readKeys.incrementAndGet();
            if (entriesFromKey.size() >= bunchSize && currentEntryList.isEmpty()) {
                // Nothing can be done. Just move on.
                lastReadKeyBytes.set(null);
                return;
            }
            if (lastReadKeyBytes.get() == null) {
                lastReadKeyBytes.set(kv.getKey());
            }
            final byte[] endKeyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(entriesFromKey.get(entriesFromKey.size() - 1).getKey()), ZERO_ARRAY);
            tr.addReadConflictRange(lastReadKeyBytes.get(), endKeyBytes);
            tr.addWriteConflictRange(lastReadKeyBytes.get(), kv.getKey());
            lastReadKeyBytes.set(endKeyBytes);
            tr.clear(kv.getKey());
            instrumentDelete(kv.getKey(), kv.getValue());
            for (Map.Entry<K, V> entry : entriesFromKey) {
                byte[] serializedEntry = serializer.serializeEntry(entry);
                if (currentEntrySize.get() + serializedEntry.length > MAX_VALUE_SIZE && !currentEntryList.isEmpty()) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
                currentEntryList.add(entry);
                currentEntrySize.addAndGet(serializedEntry.length);
                if (currentEntryList.size() == bunchSize) {
                    flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
                    currentEntrySize.set(0);
                }
            }
        }, tr.getExecutor()).thenApply(vignore -> {
            if (!currentEntryList.isEmpty()) {
                flushEntryList(tr, subspaceKey, currentEntryList, lastKey);
            }
            // Return a valid continuation if there might be more keys
            if (lastKey.get() != null && keyLimit != ReadTransaction.ROW_LIMIT_UNLIMITED && readKeys.get() == keyLimit) {
                return ByteArrayUtil.join(subspaceKey, serializer.serializeKey(lastKey.get()), ZERO_ARRAY);
            } else {
                return null;
            }
        });
    });
}

Example 15 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMap method put.

/**
 * Inserts or updates a key into a map with a new value. This will find an appropriate
 * bunch to insert the key into (or create one if one doesn't exist or if all of the candidates
 * are full). It will do work to make sure that the placement is locally optimal (that is, it
 * will choose between the one or two bunches closest to the key when performing its bunching).
 * It makes no attempt to fix suboptimal bunches elsewhere within the map. If the map already
 * contains <code>key</code>, it will overwrite the existing key with the new value. This will
 * return the old value if one is present.
 *
 * <p>
 * Note that this method is <b>not</b> thread-safe if multiple threads call it with the same
 * transaction and subspace. (Multiple calls with different transactions or subspaces are safe.)
 * </p>
 *
 * <p>
 * Note that this call is asynchronous. It will return a {@link CompletableFuture} that will be
 * completed when this task has completed.
 * </p>
 *
 * @param tcx database or transaction to use when performing the insertion
 * @param subspace subspace within which the map's data are located
 * @param key key of the map entry to insert
 * @param value value of the map entry to insert
 * @return a future that will complete with an optional that will either contain the previous value
 *         associated with the key or be empty if there was not a previous value
 */
@Nonnull
public CompletableFuture<Optional<V>> put(@Nonnull TransactionContext tcx, @Nonnull Subspace subspace, @Nonnull K key, @Nonnull V value) {
    return tcx.runAsync(tr -> {
        byte[] subspaceKey = subspace.pack();
        byte[] keyBytes = ByteArrayUtil.join(subspaceKey, serializer.serializeKey(key));
        // keys are added (within this transaction) to the RYW cache.
        return instrumentRangeRead(tr.snapshot().getRange(KeySelector.lastLessOrEqual(keyBytes), KeySelector.firstGreaterThan(keyBytes).add(1), ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList()).thenApply(keyValues -> {
            KeyValue kvBefore = null;
            KeyValue kvAfter = null;
            for (KeyValue next : keyValues) {
                if (ByteArrayUtil.startsWith(next.getKey(), subspaceKey)) {
                    if (ByteArrayUtil.compareUnsigned(keyBytes, next.getKey()) < 0) {
                        kvAfter = next;
                        // no need to continue after kvAfter is set
                        break;
                    }
                    if (ByteArrayUtil.compareUnsigned(next.getKey(), keyBytes) <= 0) {
                        kvBefore = next;
                    }
                }
            }
            // picking the correct keys and values.
            if (kvBefore != null && (ByteArrayUtil.compareUnsigned(keyBytes, kvBefore.getKey()) < 0 || !ByteArrayUtil.startsWith(kvBefore.getKey(), subspaceKey))) {
                throw new BunchedMapException("database key before map key compared incorrectly").addLogInfo(LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspaceKey)).addLogInfo("key", ByteArrayUtil2.loggable(keyBytes)).addLogInfo("kvBefore", ByteArrayUtil2.loggable(kvBefore.getKey()));
            }
            if (kvAfter != null && (ByteArrayUtil.compareUnsigned(keyBytes, kvAfter.getKey()) >= 0 || !ByteArrayUtil.startsWith(kvAfter.getKey(), subspaceKey))) {
                throw new BunchedMapException("database key after map key compared incorrectly").addLogInfo(LogMessageKeys.SUBSPACE, ByteArrayUtil2.loggable(subspaceKey)).addLogInfo("key", ByteArrayUtil2.loggable(keyBytes)).addLogInfo("kvAfter", ByteArrayUtil2.loggable(kvAfter.getKey()));
            }
            Map.Entry<K, V> newEntry = new AbstractMap.SimpleImmutableEntry<>(key, value);
            return insertEntry(tr, subspaceKey, keyBytes, key, value, kvBefore, kvAfter, newEntry);
        });
    });
}