Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 36 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class KeyValueCursorTest method limiterWithLookahead.

@Test
public void limiterWithLookahead() {
    fdb.run(context -> {
        RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(1);
        KeyValueCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace).setContext(context).setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE).setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE).setScanProperties(forwardScanWithLimiter(limiter)).build();
        // should exhaust limit first
        RecordCursor<KeyValue> cursor = kvCursor.skip(2);
        RecordCursorResult<KeyValue> result = cursor.getNext();
        assertThat("skipped items should exhaust limit", result.hasNext(), is(false));
        assertThat("no next reason should be SCAN_LIMIT_REACHED", result.getNoNextReason(), equalTo(RecordCursor.NoNextReason.SCAN_LIMIT_REACHED));
        return null;
    });
}

Example 37 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class TextIndexTest method printUsage.

private void printUsage() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openRecordStore(context);
        Subspace indexSubspace = recordStore.getIndexMaintainer(recordStore.getRecordMetaData().getIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME)).getIndexSubspace();
        final int indexSuspaceLength = indexSubspace.getKey().length;
        int subspaceOverhead = 0;
        int keySize = 0;
        int valueSize = 0;
        for (KeyValue kv : context.ensureActive().getRange(indexSubspace.range())) {
            subspaceOverhead += indexSuspaceLength;
            keySize += kv.getKey().length - indexSuspaceLength;
            valueSize += kv.getValue().length;
        }
        int textSize = 0;
        RecordCursorIterator<String> cursor = recordStore.scanRecords(null, ScanProperties.FORWARD_SCAN).map(record -> {
            Message msg = record.getRecord();
            Descriptors.FieldDescriptor fd = msg.getDescriptorForType().findFieldByName("text");
            return msg.getField(fd).toString();
        }).asIterator();
        while (cursor.hasNext()) {
            textSize += cursor.next().length();
        }
        LOGGER.info("Usage:");
        LOGGER.info("  Subspace: {} kB", subspaceOverhead * 1e-3);
        LOGGER.info("  Keys:     {} kB", keySize * 1e-3);
        LOGGER.info("  Values:   {} kB", valueSize * 1e-3);
        LOGGER.info("  Text:     {} kB", textSize * 1e-3);
        LOGGER.info("  Overhead: {}", textSize == 0.0 ? Double.POSITIVE_INFINITY : ((subspaceOverhead + keySize + valueSize) * 1.0 / textSize));
    }
}

Example 38 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method stressTest.

private void stressTest(final Random r, final int trTotal, final int opTotal, final int keyCount, final int workerCount, boolean addBytesToValue, AtomicLong globalTrCount, int mapCount) throws InterruptedException, ExecutionException {
    final long initialTrCount = globalTrCount.get();
    final Subspace logSubspace = DirectoryLayer.getDefault().createOrOpen(db, PathUtil.from(getClass().getName(), "log")).get();
    db.run(tr -> {
        tr.clear(bmSubspace.range());
        tr.clear(logSubspace.range());
        // If the database is empty, putting these here stop scans from hitting the log subspace within a transaction
        tr.set(logSubspace.getKey(), new byte[0]);
        tr.set(ByteArrayUtil.join(logSubspace.getKey(), new byte[] { (byte) 0xff }), new byte[0]);
        return null;
    });
    final List<CompletableFuture<Void>> workers = Stream.generate(() -> {
        int bunchSize = r.nextInt(15) + 1;
        BunchedMap<Tuple, Tuple> workerMap = new BunchedMap<>(serializer, Comparator.naturalOrder(), bunchSize);
        AtomicInteger trCount = new AtomicInteger(0);
        return AsyncUtil.whileTrue(() -> {
            final Transaction tr = db.createTransaction();
            tr.options().setDebugTransactionIdentifier("stress-tr-" + globalTrCount.getAndIncrement());
            tr.options().setLogTransaction();
            final AtomicInteger opCount = new AtomicInteger(0);
            final AtomicInteger localOrder = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                int opCode = r.nextInt(4);
                CompletableFuture<?> op;
                if (opCode == 0) {
                    // Random put
                    CompletableFuture<?>[] futures = new CompletableFuture<?>[mapCount];
                    for (int i = 0; i < mapCount; i++) {
                        if (r.nextBoolean()) {
                            Tuple key = Tuple.from(r.nextInt(keyCount));
                            Tuple value;
                            if (addBytesToValue) {
                                int byteLength = r.nextInt(5000);
                                byte[] bytes = new byte[byteLength];
                                r.nextBytes(bytes);
                                value = Tuple.from(r.nextLong(), bytes);
                            } else {
                                value = Tuple.from(r.nextLong());
                            }
                            tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, i, localOrder), Tuple.from("PUT", key, value).pack());
                            futures[i] = workerMap.put(tr, bmSubspace.subspace(Tuple.from(i)), key, value);
                        } else {
                            futures[i] = AsyncUtil.DONE;
                        }
                    }
                    op = CompletableFuture.allOf(futures);
                } else if (opCode == 1) {
                    // Read a random key.
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.get(tr, bmSubspace.get(mapIndex), key).thenAccept(optionalValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("GET", key, optionalValue.orElse(null)).pack()));
                } else if (opCode == 2) {
                    // Check contains key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.containsKey(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(wasPresent -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("CONTAINS_KEY", key, wasPresent).pack()));
                } else {
                    // Remove a random key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.remove(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(oldValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("REMOVE", key, oldValue.orElse(null)).pack()));
                }
                return op.thenApply(ignore -> opCount.incrementAndGet() < opTotal);
            }).thenCompose(vignore -> tr.commit()).handle((vignore, err) -> {
                tr.close();
                if (err != null) {
                    FDBException fdbE = unwrapException(err);
                    if (fdbE != null) {
                        if (fdbE.getCode() != FDBError.NOT_COMMITTED.code() && fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code()) {
                            throw fdbE;
                        }
                    } else {
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new RuntimeException("verification error", err);
                        }
                    }
                }
                return trCount.incrementAndGet() < trTotal;
            });
        });
    }).limit(workerCount).collect(Collectors.toList());
    final AtomicBoolean stillWorking = new AtomicBoolean(true);
    final CompletableFuture<Void> verifierWorker = AsyncUtil.whileTrue(() -> {
        Transaction tr = db.createTransaction();
        AtomicLong versionRef = new AtomicLong(-1L);
        return tr.getReadVersion().thenCompose(version -> {
            versionRef.set(version);
            // Grab the mutation list.
            AtomicInteger mapIndex = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                Subspace mapSubspace = bmSubspace.subspace(Tuple.from(mapIndex.get()));
                Subspace mapLogSubspace = logSubspace.subspace(Tuple.from(mapIndex.get()));
                CompletableFuture<List<Tuple>> logFuture = AsyncUtil.mapIterable(tr.getRange(mapLogSubspace.range()), kv -> Tuple.fromBytes(kv.getValue())).asList();
                // Verify integrity and then grab all of the keys and values.
                CompletableFuture<List<Map.Entry<Tuple, Tuple>>> contentFuture = AsyncUtil.collectRemaining(map.scan(tr, mapSubspace));
                CompletableFuture<Void> integrityFuture = map.verifyIntegrity(tr, mapSubspace);
                return integrityFuture.thenCompose(vignore -> contentFuture.thenCombine(logFuture, (mapContents, logEntries) -> {
                    Map<Tuple, Tuple> mapCopy = new TreeMap<>();
                    for (Tuple logEntry : logEntries) {
                        String op = logEntry.getString(0);
                        if (op.equals("PUT")) {
                            mapCopy.put(logEntry.getNestedTuple(1), logEntry.getNestedTuple(2));
                        } else if (op.equals("GET")) {
                            assertEquals(logEntry.getNestedTuple(2), mapCopy.get(logEntry.getNestedTuple(1)));
                        } else if (op.equals("CONTAINS_KEY")) {
                            assertEquals(logEntry.getBoolean(2), mapCopy.containsKey(logEntry.getNestedTuple(1)));
                        } else if (op.equals("REMOVE")) {
                            Tuple oldValue = mapCopy.remove(logEntry.getNestedTuple(1));
                            assertEquals(logEntry.getNestedTuple(2), oldValue);
                        } else {
                            fail("Unexpected operation " + op);
                        }
                    }
                    assertEquals(new ArrayList<>(mapCopy.entrySet()), mapContents);
                    return mapIndex.incrementAndGet() < mapCount;
                })).handle((res, err) -> {
                    // Report error information unless it was just a transaction timeout (in which case we'll retry).
                    FDBException fdbE = unwrapException(err);
                    if (err != null && (fdbE == null || fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code())) {
                        System.err.println("Error verifying consistency: " + err);
                        err.printStackTrace();
                        List<Map.Entry<Tuple, Tuple>> contents = contentFuture.join();
                        System.err.println("Map contents:");
                        contents.forEach(entry -> System.err.println("  " + entry.getKey() + " -> " + entry.getValue()));
                        System.err.println("DB contents:");
                        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
                        rangeKVs.forEach(kv -> {
                            Tuple boundaryKey = bmSubspace.unpack(kv.getKey());
                            System.err.println("  " + boundaryKey + " -> " + serializer.deserializeEntries(boundaryKey, kv.getValue()));
                        });
                        List<Tuple> logEntries = logFuture.join();
                        System.err.println("Log contents:");
                        logEntries.forEach(logEntry -> System.err.println(" " + logEntry));
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new LoggableException("unable to complete consistency check", err);
                        }
                    }
                    return res;
                });
            });
        }).whenComplete((v, t) -> tr.close()).thenApply(vignore -> stillWorking.get());
    });
    AtomicInteger mapIndex = new AtomicInteger(0);
    CompletableFuture<Void> compactingWorker = AsyncUtil.whileTrue(() -> {
        AtomicReference<byte[]> continuation = new AtomicReference<>(null);
        return AsyncUtil.whileTrue(() -> map.compact(db, bmSubspace.subspace(Tuple.from(mapIndex.get())), 5, continuation.get()).thenApply(nextContinuation -> {
            continuation.set(nextContinuation);
            return nextContinuation != null;
        })).thenApply(vignore -> {
            mapIndex.getAndUpdate(oldIndex -> (oldIndex + 1) % mapCount);
            return stillWorking.get();
        });
    });
    // Wait for all workers to stop working.
    AsyncUtil.whenAll(workers).whenComplete((vignore, err) -> stillWorking.set(false)).thenAcceptBoth(verifierWorker, (vignore1, vignore2) -> {
    }).thenAcceptBoth(compactingWorker, (vignore1, vignore2) -> {
    }).whenComplete((vignore, err) -> {
        System.out.printf("Completed stress test with %d workers, %d keys, and %d transactions %s (large values=%s).%n", workerCount, keyCount, globalTrCount.get() - initialTrCount, (err == null ? "successfully" : "with an error"), addBytesToValue);
        if (err != null) {
            err.printStackTrace();
        }
        for (int i = 0; i < mapCount; i++) {
            System.out.println(" Map " + i + ":");
            Subspace mapSubspace = bmSubspace.subspace(Tuple.from(i));
            List<KeyValue> rangeKVs = inconsistentScan(db, mapSubspace);
            System.out.println("  Boundary keys: " + rangeKVs.stream().map(kv -> mapSubspace.unpack(kv.getKey())).collect(Collectors.toList()));
            System.out.println("  Boundary info:");
            rangeKVs.forEach(kv -> {
                Tuple boundaryKey = mapSubspace.unpack(kv.getKey());
                System.out.printf("    %s: %d - %s%n", boundaryKey, serializer.deserializeEntries(boundaryKey, kv.getValue()).size(), serializer.deserializeKeys(boundaryKey, kv.getValue()));
            });
        }
        int opsCount = inconsistentScan(db, logSubspace).size();
        System.out.println("  Committed ops: " + opsCount);
    }).get();
}

Example 39 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method insertTwoKeys.

@Test
public void insertTwoKeys() throws ExecutionException, InterruptedException {
    final Tuple value = Tuple.from("hello", "there");
    final List<Tuple> firstTuples = LongStream.range(100L, 110L).boxed().map(Tuple::from).collect(Collectors.toList());
    final List<Tuple> secondTuples = LongStream.range(120L, 130L).boxed().map(Tuple::from).collect(Collectors.toList());
    db.run(tr -> {
        firstTuples.forEach(t -> map.put(tr, bmSubspace, t, value).join());
        secondTuples.forEach(t -> map.put(tr, bmSubspace, t, value).join());
        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
        assertEquals(2, rangeKVs.size());
        firstTuples.forEach(t -> assertTrue(map.containsKey(tr, bmSubspace, t).join(), t.toString() + " not in map"));
        secondTuples.forEach(t -> assertTrue(map.containsKey(tr, bmSubspace, t).join(), t.toString() + " not in map"));
        return null;
    });
    try (Transaction tr = db.createTransaction()) {
        // Insert in the middle.
        List<Tuple> middleTuples = Stream.of(115L, 118L, 119L, 114L).map(Tuple::from).collect(Collectors.toList());
        Tuple minSoFar = null;
        for (int i = 0; i < middleTuples.size(); i++) {
            Tuple t = middleTuples.get(i);
            map.put(tr, bmSubspace, t, value).join();
            minSoFar = (minSoFar == null || t.compareTo(minSoFar) < 0) ? t : minSoFar;
            for (int j = 0; j < middleTuples.size(); j++) {
                assertEquals(j <= i, map.containsKey(tr, bmSubspace, middleTuples.get(j)).get());
            }
            List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
            assertEquals(3, rangeKVs.size());
            List<Tuple> keys = rangeKVs.stream().map(KeyValue::getKey).map(bmSubspace::unpack).collect(Collectors.toList());
            assertEquals(Arrays.asList(Tuple.from(100L), minSoFar, Tuple.from(120L)), keys);
        }
        tr.cancel();
    }
    try (Transaction tr = db.createTransaction()) {
        // Remove a key from the end of first tuple collection.
        assertTrue(map.remove(tr, bmSubspace, Tuple.from(109L)).get().isPresent());
        assertFalse(map.remove(tr, bmSubspace, Tuple.from(109L)).get().isPresent());
        map.put(tr, bmSubspace, Tuple.from(110L), value).get();
        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().get();
        assertEquals(2, rangeKVs.size());
        // Now insert in the middle to force it to split.
        map.put(tr, bmSubspace, Tuple.from(109L), value).get();
        rangeKVs = tr.getRange(bmSubspace.range()).asList().get();
        assertEquals(3, rangeKVs.size());
        List<Tuple> keys = rangeKVs.stream().map(KeyValue::getKey).map(bmSubspace::unpack).collect(Collectors.toList());
        assertEquals(Arrays.asList(Tuple.from(100L), Tuple.from(105L), Tuple.from(120L)), keys);
        tr.cancel();
    }
}

Example 40 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method verifyBoundaryKeys.

private void verifyBoundaryKeys(@Nonnull List<Tuple> boundaryKeys) throws ExecutionException, InterruptedException {
    try (Transaction tr = db.createTransaction()) {
        map.verifyIntegrity(tr, bmSubspace).get();
        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().get();
        List<Tuple> actualBoundaryKeys = rangeKVs.stream().map(KeyValue::getKey).map(bmSubspace::unpack).collect(Collectors.toList());
        List<Map.Entry<Tuple, Tuple>> entryList = rangeKVs.stream().flatMap(kv -> serializer.deserializeEntries(bmSubspace.unpack(kv.getKey()), kv.getValue()).stream()).collect(Collectors.toList());
        System.out.println(entryList);
        assertEquals(boundaryKeys, actualBoundaryKeys);
        tr.cancel();
    }
}