Examples with FDBException com.apple.foundationdb.FDBException used on opensource projects

Example 21 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class FDBRecordStoreCrudTest method readMissingPreloaded.

@Test
public void readMissingPreloaded() throws Exception {
    try (FDBRecordContext context = openContext()) {
        openSimpleRecordStore(context);
        // 4488 does not exist
        // ensure loaded in context
        recordStore.preloadRecordAsync(Tuple.from(4488L)).get();
        // ensure no more I/O done through the transaction
        context.ensureActive().cancel();
        FDBStoredRecord<Message> record = recordStore.loadRecord(Tuple.from(4488L));
        assertNull(record);
        FDBExceptions.FDBStoreException e = assertThrows(FDBExceptions.FDBStoreException.class, context::commit);
        assertNotNull(e.getCause());
        assertThat(e.getCause(), instanceOf(FDBException.class));
        FDBException fdbE = (FDBException) e.getCause();
        assertEquals(FDBError.TRANSACTION_CANCELLED.code(), fdbE.getCode());
    }
}

Example 22 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method notReincreaseLimit.

@Test
void notReincreaseLimit() {
    // Non-retriable error that is in lessen work codes.
    Supplier<RuntimeException> createException = () -> new RecordCoreException("Non-retriable", new FDBException("transaction_too_large", 2101));
    Queue<Pair<Integer, Supplier<RuntimeException>>> queue = new LinkedList<>();
    // failures until it hits 42
    for (int i = 100; i > 42; i = (3 * i) / 4) {
        queue.add(Pair.of(i, createException));
    }
    // a whole bunch of successes
    for (int i = 0; i < 100; i++) {
        queue.add(Pair.of(42, null));
    }
    reincreaseLimit(queue, index -> OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setLimit(100).setMaxRetries(queue.size() + 3).setRecordsPerSecond(10000).setMdcContext(ImmutableMap.of("mdcKey", "my cool mdc value")).setMaxAttempts(3).build());
}

Example 23 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method run.

@Test
public void run() {
    Index index = runAsyncSetup();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setLimit(100).setMaxRetries(3).setRecordsPerSecond(10000).setMdcContext(ImmutableMap.of("mdcKey", "my cool mdc value")).setMaxAttempts(2).build()) {
        AtomicInteger attempts = new AtomicInteger();
        // Non-FDB error
        attempts.set(0);
        runAndHandleLessenWorkCodes(indexBuilder, store -> {
            attempts.incrementAndGet();
            throw new IllegalStateException("illegal state");
        }).handle((val, e) -> {
            assertNotNull(e);
            assertThat(e, instanceOf(IllegalStateException.class));
            assertEquals("illegal state", e.getMessage());
            assertNull(e.getCause());
            assertEquals(1, attempts.get());
            assertEquals("my cool mdc value", ThreadContext.get("mdcKey"));
            return null;
        }).join();
        // Retriable error that is not in lessen work codes.
        attempts.set(0);
        runAndHandleLessenWorkCodes(indexBuilder, store -> {
            attempts.incrementAndGet();
            throw new RecordCoreRetriableTransactionException("Retriable", new FDBException("commit_unknown_result", 1021));
        }).handle((val, e) -> {
            assertNotNull(e);
            assertThat(e, instanceOf(RecordCoreRetriableTransactionException.class));
            assertEquals("Retriable", e.getMessage());
            assertThat(e.getCause(), instanceOf(FDBException.class));
            assertEquals("commit_unknown_result", e.getCause().getMessage());
            assertEquals(FDBError.COMMIT_UNKNOWN_RESULT.code(), ((FDBException) e.getCause()).getCode());
            assertEquals(2, attempts.get());
            assertEquals("my cool mdc value", ThreadContext.get("mdcKey"));
            return null;
        }).join();
        // Non-retriable error that is in lessen work codes.
        attempts.set(0);
        runAndHandleLessenWorkCodes(indexBuilder, store -> {
            attempts.incrementAndGet();
            throw new RecordCoreException("Non-retriable", new FDBException("transaction_too_large", 2101));
        }).handle((val, e) -> {
            assertNotNull(e);
            assertThat(e, instanceOf(RecordCoreException.class));
            assertEquals("Non-retriable", e.getMessage());
            assertNotNull(e.getCause());
            assertThat(e.getCause(), instanceOf(FDBException.class));
            assertEquals("transaction_too_large", e.getCause().getMessage());
            assertEquals(FDBError.TRANSACTION_TOO_LARGE.code(), ((FDBException) e.getCause()).getCode());
            // lessenWorkCodes is maxRetries
            assertEquals(4, attempts.get());
            assertEquals("my cool mdc value", ThreadContext.get("mdcKey"));
            return null;
        }).join();
        // Retriable error that is in lessen work codes.
        attempts.set(0);
        runAndHandleLessenWorkCodes(indexBuilder, store -> {
            attempts.incrementAndGet();
            throw new RecordCoreRetriableTransactionException("Retriable and lessener", new FDBException("not_committed", 1020));
        }).handle((val, e) -> {
            assertNotNull(e);
            assertThat(e, instanceOf(RecordCoreRetriableTransactionException.class));
            assertEquals("Retriable and lessener", e.getMessage());
            assertNotNull(e.getCause());
            assertThat(e.getCause(), instanceOf(FDBException.class));
            assertEquals("not_committed", e.getCause().getMessage());
            assertEquals(FDBError.NOT_COMMITTED.code(), ((FDBException) e.getCause()).getCode());
            assertEquals(8, attempts.get());
            assertEquals("my cool mdc value", ThreadContext.get("mdcKey"));
            return null;
        }).join();
    }
}

Example 24 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class OnlineIndexerSimpleTest method recordsScanned.

@Test
public void recordsScanned() {
    Supplier<RuntimeException> nonRetriableException = () -> new RecordCoreException("Non-retriable", new FDBException("transaction_too_large", 2101));
    Supplier<RuntimeException> retriableException = () -> new RecordCoreRetriableTransactionException("Retriable", new FDBException("not_committed", 1020));
    Queue<Pair<Long, Supplier<RuntimeException>>> queue = new LinkedList<>();
    queue.add(Pair.of(0L, retriableException));
    queue.add(Pair.of(0L, nonRetriableException));
    queue.add(Pair.of(0L, null));
    queue.add(Pair.of(1L, null));
    queue.add(Pair.of(2L, null));
    queue.add(Pair.of(3L, null));
    queue.add(Pair.of(4L, retriableException));
    queue.add(Pair.of(4L, retriableException));
    queue.add(Pair.of(4L, nonRetriableException));
    queue.add(Pair.of(4L, nonRetriableException));
    queue.add(Pair.of(4L, null));
    Index index = runAsyncSetup();
    try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder().setDatabase(fdb).setMetaData(metaData).setIndex(index).setSubspace(subspace).setMdcContext(ImmutableMap.of("mdcKey", "my cool mdc value")).setMaxAttempts(3).setConfigLoader(old -> OnlineIndexer.Config.newBuilder().setMaxLimit(100).setMaxRetries(queue.size() + 3).setRecordsPerSecond(10000).setIncreaseLimitAfter(10).setProgressLogIntervalMillis(30).build()).build()) {
        AtomicInteger attempts = new AtomicInteger();
        attempts.set(0);
        AsyncUtil.whileTrue(() -> indexBuilder.buildCommitRetryAsync((store, recordsScanned) -> {
            Pair<Long, Supplier<RuntimeException>> behavior = queue.poll();
            if (behavior == null) {
                return AsyncUtil.READY_FALSE;
            } else {
                int currentAttempt = attempts.getAndIncrement();
                assertEquals(1, recordsScanned.incrementAndGet());
                assertEquals(behavior.getLeft().longValue(), indexBuilder.getTotalRecordsScanned(), "Attempt " + currentAttempt);
                if (behavior.getRight() != null) {
                    throw behavior.getRight().get();
                }
                return AsyncUtil.READY_TRUE;
            }
        }, Arrays.asList(LogMessageKeys.CALLING_METHOD, "OnlineIndexerTest.recordsScanned"))).join();
        assertNull(queue.poll());
        assertEquals(5L, indexBuilder.getTotalRecordsScanned());
    }
}

Example 25 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method stressTest.

private void stressTest(final Random r, final int trTotal, final int opTotal, final int keyCount, final int workerCount, boolean addBytesToValue, AtomicLong globalTrCount, int mapCount) throws InterruptedException, ExecutionException {
    final long initialTrCount = globalTrCount.get();
    final Subspace logSubspace = DirectoryLayer.getDefault().createOrOpen(db, PathUtil.from(getClass().getName(), "log")).get();
    db.run(tr -> {
        tr.clear(bmSubspace.range());
        tr.clear(logSubspace.range());
        // If the database is empty, putting these here stop scans from hitting the log subspace within a transaction
        tr.set(logSubspace.getKey(), new byte[0]);
        tr.set(ByteArrayUtil.join(logSubspace.getKey(), new byte[] { (byte) 0xff }), new byte[0]);
        return null;
    });
    final List<CompletableFuture<Void>> workers = Stream.generate(() -> {
        int bunchSize = r.nextInt(15) + 1;
        BunchedMap<Tuple, Tuple> workerMap = new BunchedMap<>(serializer, Comparator.naturalOrder(), bunchSize);
        AtomicInteger trCount = new AtomicInteger(0);
        return AsyncUtil.whileTrue(() -> {
            final Transaction tr = db.createTransaction();
            tr.options().setDebugTransactionIdentifier("stress-tr-" + globalTrCount.getAndIncrement());
            tr.options().setLogTransaction();
            final AtomicInteger opCount = new AtomicInteger(0);
            final AtomicInteger localOrder = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                int opCode = r.nextInt(4);
                CompletableFuture<?> op;
                if (opCode == 0) {
                    // Random put
                    CompletableFuture<?>[] futures = new CompletableFuture<?>[mapCount];
                    for (int i = 0; i < mapCount; i++) {
                        if (r.nextBoolean()) {
                            Tuple key = Tuple.from(r.nextInt(keyCount));
                            Tuple value;
                            if (addBytesToValue) {
                                int byteLength = r.nextInt(5000);
                                byte[] bytes = new byte[byteLength];
                                r.nextBytes(bytes);
                                value = Tuple.from(r.nextLong(), bytes);
                            } else {
                                value = Tuple.from(r.nextLong());
                            }
                            tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, i, localOrder), Tuple.from("PUT", key, value).pack());
                            futures[i] = workerMap.put(tr, bmSubspace.subspace(Tuple.from(i)), key, value);
                        } else {
                            futures[i] = AsyncUtil.DONE;
                        }
                    }
                    op = CompletableFuture.allOf(futures);
                } else if (opCode == 1) {
                    // Read a random key.
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.get(tr, bmSubspace.get(mapIndex), key).thenAccept(optionalValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("GET", key, optionalValue.orElse(null)).pack()));
                } else if (opCode == 2) {
                    // Check contains key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.containsKey(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(wasPresent -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("CONTAINS_KEY", key, wasPresent).pack()));
                } else {
                    // Remove a random key
                    int mapIndex = r.nextInt(mapCount);
                    Tuple key = Tuple.from(r.nextInt(keyCount));
                    op = workerMap.remove(tr, bmSubspace.subspace(Tuple.from(mapIndex)), key).thenAccept(oldValue -> tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, getLogKey(logSubspace, mapIndex, localOrder), Tuple.from("REMOVE", key, oldValue.orElse(null)).pack()));
                }
                return op.thenApply(ignore -> opCount.incrementAndGet() < opTotal);
            }).thenCompose(vignore -> tr.commit()).handle((vignore, err) -> {
                tr.close();
                if (err != null) {
                    FDBException fdbE = unwrapException(err);
                    if (fdbE != null) {
                        if (fdbE.getCode() != FDBError.NOT_COMMITTED.code() && fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code()) {
                            throw fdbE;
                        }
                    } else {
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new RuntimeException("verification error", err);
                        }
                    }
                }
                return trCount.incrementAndGet() < trTotal;
            });
        });
    }).limit(workerCount).collect(Collectors.toList());
    final AtomicBoolean stillWorking = new AtomicBoolean(true);
    final CompletableFuture<Void> verifierWorker = AsyncUtil.whileTrue(() -> {
        Transaction tr = db.createTransaction();
        AtomicLong versionRef = new AtomicLong(-1L);
        return tr.getReadVersion().thenCompose(version -> {
            versionRef.set(version);
            // Grab the mutation list.
            AtomicInteger mapIndex = new AtomicInteger(0);
            return AsyncUtil.whileTrue(() -> {
                Subspace mapSubspace = bmSubspace.subspace(Tuple.from(mapIndex.get()));
                Subspace mapLogSubspace = logSubspace.subspace(Tuple.from(mapIndex.get()));
                CompletableFuture<List<Tuple>> logFuture = AsyncUtil.mapIterable(tr.getRange(mapLogSubspace.range()), kv -> Tuple.fromBytes(kv.getValue())).asList();
                // Verify integrity and then grab all of the keys and values.
                CompletableFuture<List<Map.Entry<Tuple, Tuple>>> contentFuture = AsyncUtil.collectRemaining(map.scan(tr, mapSubspace));
                CompletableFuture<Void> integrityFuture = map.verifyIntegrity(tr, mapSubspace);
                return integrityFuture.thenCompose(vignore -> contentFuture.thenCombine(logFuture, (mapContents, logEntries) -> {
                    Map<Tuple, Tuple> mapCopy = new TreeMap<>();
                    for (Tuple logEntry : logEntries) {
                        String op = logEntry.getString(0);
                        if (op.equals("PUT")) {
                            mapCopy.put(logEntry.getNestedTuple(1), logEntry.getNestedTuple(2));
                        } else if (op.equals("GET")) {
                            assertEquals(logEntry.getNestedTuple(2), mapCopy.get(logEntry.getNestedTuple(1)));
                        } else if (op.equals("CONTAINS_KEY")) {
                            assertEquals(logEntry.getBoolean(2), mapCopy.containsKey(logEntry.getNestedTuple(1)));
                        } else if (op.equals("REMOVE")) {
                            Tuple oldValue = mapCopy.remove(logEntry.getNestedTuple(1));
                            assertEquals(logEntry.getNestedTuple(2), oldValue);
                        } else {
                            fail("Unexpected operation " + op);
                        }
                    }
                    assertEquals(new ArrayList<>(mapCopy.entrySet()), mapContents);
                    return mapIndex.incrementAndGet() < mapCount;
                })).handle((res, err) -> {
                    // Report error information unless it was just a transaction timeout (in which case we'll retry).
                    FDBException fdbE = unwrapException(err);
                    if (err != null && (fdbE == null || fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code())) {
                        System.err.println("Error verifying consistency: " + err);
                        err.printStackTrace();
                        List<Map.Entry<Tuple, Tuple>> contents = contentFuture.join();
                        System.err.println("Map contents:");
                        contents.forEach(entry -> System.err.println("  " + entry.getKey() + " -> " + entry.getValue()));
                        System.err.println("DB contents:");
                        List<KeyValue> rangeKVs = tr.getRange(bmSubspace.range()).asList().join();
                        rangeKVs.forEach(kv -> {
                            Tuple boundaryKey = bmSubspace.unpack(kv.getKey());
                            System.err.println("  " + boundaryKey + " -> " + serializer.deserializeEntries(boundaryKey, kv.getValue()));
                        });
                        List<Tuple> logEntries = logFuture.join();
                        System.err.println("Log contents:");
                        logEntries.forEach(logEntry -> System.err.println(" " + logEntry));
                        if (err instanceof RuntimeException) {
                            throw (RuntimeException) err;
                        } else {
                            throw new LoggableException("unable to complete consistency check", err);
                        }
                    }
                    return res;
                });
            });
        }).whenComplete((v, t) -> tr.close()).thenApply(vignore -> stillWorking.get());
    });
    AtomicInteger mapIndex = new AtomicInteger(0);
    CompletableFuture<Void> compactingWorker = AsyncUtil.whileTrue(() -> {
        AtomicReference<byte[]> continuation = new AtomicReference<>(null);
        return AsyncUtil.whileTrue(() -> map.compact(db, bmSubspace.subspace(Tuple.from(mapIndex.get())), 5, continuation.get()).thenApply(nextContinuation -> {
            continuation.set(nextContinuation);
            return nextContinuation != null;
        })).thenApply(vignore -> {
            mapIndex.getAndUpdate(oldIndex -> (oldIndex + 1) % mapCount);
            return stillWorking.get();
        });
    });
    // Wait for all workers to stop working.
    AsyncUtil.whenAll(workers).whenComplete((vignore, err) -> stillWorking.set(false)).thenAcceptBoth(verifierWorker, (vignore1, vignore2) -> {
    }).thenAcceptBoth(compactingWorker, (vignore1, vignore2) -> {
    }).whenComplete((vignore, err) -> {
        System.out.printf("Completed stress test with %d workers, %d keys, and %d transactions %s (large values=%s).%n", workerCount, keyCount, globalTrCount.get() - initialTrCount, (err == null ? "successfully" : "with an error"), addBytesToValue);
        if (err != null) {
            err.printStackTrace();
        }
        for (int i = 0; i < mapCount; i++) {
            System.out.println(" Map " + i + ":");
            Subspace mapSubspace = bmSubspace.subspace(Tuple.from(i));
            List<KeyValue> rangeKVs = inconsistentScan(db, mapSubspace);
            System.out.println("  Boundary keys: " + rangeKVs.stream().map(kv -> mapSubspace.unpack(kv.getKey())).collect(Collectors.toList()));
            System.out.println("  Boundary info:");
            rangeKVs.forEach(kv -> {
                Tuple boundaryKey = mapSubspace.unpack(kv.getKey());
                System.out.printf("    %s: %d - %s%n", boundaryKey, serializer.deserializeEntries(boundaryKey, kv.getValue()).size(), serializer.deserializeKeys(boundaryKey, kv.getValue()));
            });
        }
        int opsCount = inconsistentScan(db, logSubspace).size();
        System.out.println("  Committed ops: " + opsCount);
    }).get();
}