Examples with KeyValue com.apple.foundationdb.KeyValue used on opensource projects

Example 21 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class HighContentionAllocatorTest method validateAllocation.

private void validateAllocation(FDBRecordContext context, HighContentionAllocator hca, Map<Long, String> allocations) {
    Subspace allocationSubspace = hca.getAllocationSubspace();
    Transaction transaction = context.ensureActive();
    List<KeyValue> keyValueList = transaction.getRange(allocationSubspace.range()).asList().join();
    Map<Long, String> storedAllocations = keyValueList.stream().collect(Collectors.toMap(kv -> extractKey(allocationSubspace, kv), this::extractValue));
    assertThat("we see the allocated keys in the subspace", allocations.entrySet(), containsInAnyOrder(storedAllocations.entrySet().toArray()));
}

Example 22 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class HighContentionAllocatorTest method testCheckForRootConflicts.

@Test
@Tag(Tags.WipesFDB)
void testCheckForRootConflicts() {
    Range everything = new Range(new byte[] { (byte) 0x00 }, new byte[] { (byte) 0xFF });
    database.run(context -> {
        context.ensureActive().clear(everything);
        return null;
    });
    try (FDBRecordContext context = database.openContext()) {
        HighContentionAllocator hca = HighContentionAllocator.forRoot(context, keySpace.path("test-path"));
        // so write something for all of those keys
        for (int i = 0; i < 64; i++) {
            byte[] key = Tuple.from(i, "string-" + i).pack();
            byte[] value = new byte[0];
            context.ensureActive().set(key, value);
        }
        try {
            hca.allocate("some-string").join();
            fail("allocate should fail in the same transaction");
        } catch (Exception e) {
            assertThat("a", e.getCause().getMessage(), is("database already has keys in allocation range"));
        }
        // check that the hca marks these keys as invalid
        // the thing here is that when the post allocation hook fails the allocator still writes a key,
        // that's actually a good thing since it will prevent us from trying to allocate that key again
        // but we need to make sure we have a way to exclude from the reverse lookup
        Subspace allocationSubspace = hca.getAllocationSubspace();
        Range initialwindow = new Range(allocationSubspace.getKey(), allocationSubspace.pack(64));
        List<KeyValue> allocatedValues = context.ensureActive().getRange(initialwindow).asList().join();
        byte[] valueBytes = allocatedValues.get(0).getValue();
        assertThat("there's exactly one allocation key", allocatedValues, hasSize(1));
        assertArrayEquals(valueBytes, new byte[] { (byte) 0xFD }, "the value is set to the magic byte");
        context.commit();
    }
    try (FDBRecordContext context = database.openContext()) {
        HighContentionAllocator hca = HighContentionAllocator.forRoot(context, keySpace.path("test-path"));
        try {
            hca.allocate("some-string").join();
            fail("allocate should fail in new transaction");
        } catch (Exception e) {
            assertThat("a", e.getCause().getMessage(), is("database already has keys in allocation range"));
        }
    }
    database.run(context -> {
        context.ensureActive().clear(everything);
        return null;
    });
}

Example 23 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class FDBDirectory method listAllInternal.

private List<String> listAllInternal() {
    // A private form of the method to allow easier instrumentation
    List<String> outList = new ArrayList<>();
    List<String> displayList = null;
    long totalSize = 0L;
    long actualTotalSize = 0L;
    for (KeyValue kv : context.ensureActive().getRange(metaSubspace.range())) {
        String name = metaSubspace.unpack(kv.getKey()).getString(0);
        outList.add(name);
        final FDBLuceneFileReference fileReference = FDBLuceneFileReference.parseFromBytes(LuceneSerializer.decode(kv.getValue()));
        // Only composite files are prefetched.
        if (name.endsWith(".cfs")) {
            try {
                readBlock(name, CompletableFuture.completedFuture(fileReference), 0);
            } catch (RecordCoreException e) {
                LOGGER.warn(getLogMessage("Exception thrown during prefetch", LogMessageKeys.FILE_NAME, name));
            }
        }
        this.fileReferenceCache.put(name, fileReference);
        if (LOGGER.isDebugEnabled()) {
            if (displayList == null) {
                displayList = new ArrayList<>();
            }
            if (kv.getValue() != null) {
                displayList.add(name + "(" + fileReference.getSize() + ")");
                totalSize += fileReference.getSize();
                actualTotalSize += fileReference.getActualSize();
            }
        }
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(getLogMessage("listAllFiles", LogMessageKeys.FILE_COUNT, outList.size(), LogMessageKeys.FILE_LIST, displayList, LogMessageKeys.FILE_TOTAL_SIZE, totalSize, LogMessageKeys.FILE_ACTUAL_TOTAL_SIZE, actualTotalSize));
    }
    return outList;
}

Example 24 with KeyValue

use of com.apple.foundationdb.KeyValue in project fdb-record-layer by FoundationDB.

the class KeyValueCursorTest method inclusiveNull.

@Test
public void inclusiveNull() {
    fdb.run(context -> {
        RecordCursorIterator<KeyValue> cursor = KeyValueCursor.Builder.withSubspace(subspace).setContext(context).setLow(Tuple.from(4), EndpointType.RANGE_INCLUSIVE).setHigh((Tuple) null, EndpointType.RANGE_INCLUSIVE).setContinuation(null).setScanProperties(ScanProperties.FORWARD_SCAN).build().asIterator();
        for (int j = 0; j < 5; j++) {
            KeyValue kv = cursor.next();
            assertArrayEquals(subspace.pack(Tuple.from(4, j)), kv.getKey());
            assertArrayEquals(Tuple.from(4, j).pack(), kv.getValue());
        }
        assertThat(cursor.hasNext(), is(false));
        return null;
    });
}

Example 25 with KeyValue

use of com.apple.foundationdb.KeyValue in project lionrock by panghy.

the class FoundationDbGrpcFacade method execute.

@Override
public void execute(DatabaseRequest request, StreamObserver<DatabaseResponse> responseObserver) {
    // fill-in additional details to the overall span
    Span overallSpan = tracer.currentSpan();
    if (overallSpan != null) {
        overallSpan.tag("client", request.getClientIdentifier()).tag("database_name", request.getDatabaseName()).tag("name", request.getName());
    }
    if (logger.isDebugEnabled()) {
        String msg = "Executing DatabaseRequest from: " + request.getClientIdentifier() + " on database: " + request.getDatabaseName() + " named: " + request.getName();
        logger.debug(msg);
        if (overallSpan != null) {
            overallSpan.event(msg);
        }
    }
    Database db = databaseMap.get(request.getDatabaseName());
    if (db == null) {
        StatusRuntimeException toThrow = Status.INVALID_ARGUMENT.withDescription("cannot find database named: " + request.getDatabaseName()).asRuntimeException();
        synchronized (responseObserver) {
            responseObserver.onError(toThrow);
        }
        throw toThrow;
    }
    Context rpcContext = Context.current();
    if (request.hasGetValue()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_value");
        }
        if (logger.isDebugEnabled()) {
            String msg = "GetValueRequest on: " + printable(request.getGetValue().getKey().toByteArray());
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            return tx.get(request.getGetValue().getKey().toByteArray());
        }), overallSpan, responseObserver, "failed to get key").thenAccept(val -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetValueResponse.Builder build = GetValueResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetValueRequest on: " + printable(request.getGetValue().getKey().toByteArray()) + " is: " + printable(val);
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (val != null) {
                    build.setValue(ByteString.copyFrom(val));
                }
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetValue(build.build()).build());
                    responseObserver.onCompleted();
                }
            }
        });
    } else if (request.hasGetKey()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_key");
        }
        io.github.panghy.lionrock.proto.KeySelector ks = request.getGetKey().getKeySelector();
        KeySelector keySelector = new KeySelector(ks.getKey().toByteArray(), ks.getOrEqual(), ks.getOffset());
        if (logger.isDebugEnabled()) {
            String msg = "GetKey for: " + keySelector;
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            return tx.getKey(keySelector);
        }), overallSpan, responseObserver, "failed to get key").thenAccept(val -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetKeyResponse.Builder build = GetKeyResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetKey on: " + keySelector + " is: " + printable(val);
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (val != null) {
                    build.setKey(ByteString.copyFrom(val));
                }
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetKey(build.build()).build());
                    responseObserver.onCompleted();
                }
            }
        });
    } else if (request.hasSetValue()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "set_value");
        }
        if (logger.isDebugEnabled()) {
            String msg = "SetValueRequest on: " + printable(request.getSetValue().getKey().toByteArray()) + " => " + printable(request.getSetValue().getValue().toByteArray());
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleCommittedTransaction(handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            tx.set(request.getSetValue().getKey().toByteArray(), request.getSetValue().getValue().toByteArray());
            return completedFuture(tx);
        }), overallSpan, responseObserver, "failed to set key"), responseObserver);
    } else if (request.hasClearKey()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "clear_key");
        }
        if (logger.isDebugEnabled()) {
            String msg = "ClearKeyRequest on: " + printable(request.getClearKey().getKey().toByteArray());
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
            logger.debug(msg);
        }
        handleCommittedTransaction(handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            tx.clear(request.getClearKey().getKey().toByteArray());
            return completedFuture(tx);
        }), overallSpan, responseObserver, "failed to set key"), responseObserver);
    } else if (request.hasClearRange()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "clear_range");
        }
        if (logger.isDebugEnabled()) {
            String msg = "ClearRangeRequest on: " + printable(request.getClearRange().getStart().toByteArray()) + " => " + printable(request.getClearRange().getEnd().toByteArray());
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleCommittedTransaction(handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            tx.clear(request.getClearRange().getStart().toByteArray(), request.getClearRange().getEnd().toByteArray());
            return completedFuture(tx);
        }), overallSpan, responseObserver, "failed to set key"), responseObserver);
    } else if (request.hasGetRange()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_range");
        }
        GetRangeRequest req = request.getGetRange();
        KeySelector start;
        if (req.hasStartBytes()) {
            start = new KeySelector(req.getStartBytes().toByteArray(), true, 0);
        } else if (req.hasStartKeySelector()) {
            start = new KeySelector(req.getStartKeySelector().getKey().toByteArray(), req.getStartKeySelector().getOrEqual(), req.getStartKeySelector().getOffset());
        } else {
            throw Status.INVALID_ARGUMENT.withDescription("must have start").asRuntimeException();
        }
        KeySelector end;
        if (req.hasEndBytes()) {
            end = new KeySelector(req.getEndBytes().toByteArray(), true, 0);
        } else if (req.hasEndKeySelector()) {
            end = new KeySelector(req.getEndKeySelector().getKey().toByteArray(), req.getEndKeySelector().getOrEqual(), req.getEndKeySelector().getOffset());
        } else {
            throw Status.INVALID_ARGUMENT.withDescription("must have end").asRuntimeException();
        }
        if (logger.isDebugEnabled()) {
            String msg = "GetRangeRequest from: " + start + " to: " + end + " reverse: " + req.getReverse() + " limit: " + req.getLimit() + " mode: " + req.getStreamingMode();
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        StreamingMode mode = StreamingMode.ITERATOR;
        switch(req.getStreamingMode()) {
            case WANT_ALL:
                mode = StreamingMode.WANT_ALL;
                break;
            case EXACT:
                mode = StreamingMode.EXACT;
                break;
        }
        StreamingMode finalMode = mode;
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            if (request.getReadVersion() > 0) {
                tx.setReadVersion(request.getReadVersion());
            }
            return tx.getRange(start, end, req.getLimit(), req.getReverse(), finalMode).asList();
        }), overallSpan, responseObserver, "failed to get range").thenAccept(results -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetRangeResponse.Builder build = GetRangeResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetRangeRequest from: " + start + " to: " + end + " reverse: " + req.getReverse() + " limit: " + req.getLimit() + " mode: " + req.getStreamingMode() + " got: " + results.size() + " rows";
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (overallSpan != null) {
                    overallSpan.tag("rows", String.valueOf(results.size()));
                }
                List<io.github.panghy.lionrock.proto.KeyValue> keyValues = new ArrayList<>(results.size());
                for (KeyValue result : results) {
                    keyValues.add(io.github.panghy.lionrock.proto.KeyValue.newBuilder().setKey(ByteString.copyFrom(result.getKey())).setValue(ByteString.copyFrom(result.getValue())).build());
                }
                build.addAllKeyValues(keyValues);
                build.setDone(true);
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetRange(build.build()).build());
                    responseObserver.onCompleted();
                }
            }
        });
    } else if (request.hasMutateValue()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "mutate_value");
        }
        if (logger.isDebugEnabled()) {
            String msg = "MutateValueRequest on: " + printable(request.getMutateValue().getKey().toByteArray()) + " => " + printable(request.getMutateValue().getParam().toByteArray()) + " with: " + request.getMutateValue().getType();
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleCommittedTransaction(handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            tx.mutate(getMutationType(request.getMutateValue().getType()), request.getMutateValue().getKey().toByteArray(), request.getMutateValue().getParam().toByteArray());
            return completedFuture(tx);
        }), overallSpan, responseObserver, "failed to mutate key"), responseObserver);
    } else if (request.hasGetEstimatedRangeSize()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_estimated_range_size");
        }
        byte[] startB = request.getGetEstimatedRangeSize().getStart().toByteArray();
        byte[] endB = request.getGetEstimatedRangeSize().getEnd().toByteArray();
        if (logger.isDebugEnabled()) {
            String msg = "GetEstimatedRangeSize for start: " + printable(startB) + " end: " + printable(endB);
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            return tx.getEstimatedRangeSizeBytes(startB, endB);
        }), overallSpan, responseObserver, "failed to get estimated range size").thenAccept(val -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetEstimatedRangeSizeResponse.Builder build = GetEstimatedRangeSizeResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetEstimatedRangeSize for start: " + printable(startB) + " end: " + printable(endB) + "is: " + val;
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (val != null) {
                    build.setSize(val);
                }
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetEstimatedRangeSize(build.build()).build());
                    responseObserver.onCompleted();
                }
            }
        });
    } else if (request.hasGetBoundaryKeys()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_boundary_keys");
        }
        GetBoundaryKeysRequest req = request.getGetBoundaryKeys();
        byte[] startB = req.getStart().toByteArray();
        byte[] endB = req.getEnd().toByteArray();
        if (logger.isDebugEnabled()) {
            String msg = "GetBoundaryKeysRequest from: " + printable(startB) + " to: " + printable(endB);
            logger.debug(msg);
            if (overallSpan != null) {
                overallSpan.event(msg);
            }
        }
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            if (request.getReadVersion() > 0) {
                tx.setReadVersion(request.getReadVersion());
            }
            CloseableAsyncIterator<byte[]> boundaryKeys = LocalityUtil.getBoundaryKeys(tx, startB, endB);
            return AsyncUtil.collectRemaining(boundaryKeys).whenComplete((bytes, throwable) -> boundaryKeys.close());
        }), overallSpan, responseObserver, "failed to get boundary keys").thenAccept(results -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetBoundaryKeysResponse.Builder build = GetBoundaryKeysResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetBoundaryKeysRequest from: " + printable(startB) + " to: " + printable(endB) + " got: " + results.size() + " rows";
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (overallSpan != null) {
                    overallSpan.tag("rows", String.valueOf(results.size()));
                }
                build.setDone(true);
                build.addAllKeys(results.stream().map(ByteString::copyFrom).collect(Collectors.toList()));
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetBoundaryKeys(build.build()).build());
                    responseObserver.onCompleted();
                }
            }
        });
    } else if (request.hasGetAddressesForKey()) {
        if (overallSpan != null) {
            overallSpan.tag("request", "get_addresses_for_key");
        }
        handleException(db.runAsync(tx -> {
            prepareTx(request, rpcContext, tx);
            return LocalityUtil.getAddressesForKey(tx, request.getGetAddressesForKey().getKey().toByteArray());
        }), overallSpan, responseObserver, "failed to get key").thenAccept(val -> {
            try (Tracer.SpanInScope ignored = tracer.withSpan(overallSpan)) {
                GetAddressesForKeyResponse.Builder build = GetAddressesForKeyResponse.newBuilder();
                if (logger.isDebugEnabled()) {
                    String msg = "GetAddressesForKey for: " + printable(request.getGetAddressesForKey().getKey().toByteArray()) + " is: " + Joiner.on(",").join(val);
                    logger.debug(msg);
                    if (overallSpan != null) {
                        overallSpan.event(msg);
                    }
                }
                if (val != null) {
                    build.addAllAddresses(Arrays.stream(val).collect(Collectors.toList()));
                }
                synchronized (responseObserver) {
                    responseObserver.onNext(DatabaseResponse.newBuilder().setGetAddresssesForKey(build).build());
                    responseObserver.onCompleted();
                }
            }
        });
    }
}