Search in sources :

Example 1 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class IndexingThrottle method throttledRunAsync.

@Nonnull
<R> CompletableFuture<R> throttledRunAsync(@Nonnull final Function<FDBRecordStore, CompletableFuture<R>> function, @Nonnull final BiFunction<R, Throwable, Pair<R, Throwable>> handlePostTransaction, @Nullable final BiConsumer<FDBException, List<Object>> handleLessenWork, @Nullable final List<Object> additionalLogMessageKeyValues) {
    List<Object> onlineIndexerLogMessageKeyValues = new ArrayList<>(Arrays.asList(LogMessageKeys.INDEX_NAME, common.getTargetIndexesNames(), LogMessageKeys.INDEXER_ID, common.getUuid()));
    if (additionalLogMessageKeyValues != null) {
        onlineIndexerLogMessageKeyValues.addAll(additionalLogMessageKeyValues);
    }
    AtomicInteger tries = new AtomicInteger(0);
    CompletableFuture<R> ret = new CompletableFuture<>();
    AtomicLong toWait = new AtomicLong(common.getRunner().getDatabase().getFactory().getInitialDelayMillis());
    AsyncUtil.whileTrue(() -> {
        loadConfig();
        return common.getRunner().runAsync(context -> common.getRecordStoreBuilder().copyBuilder().setContext(context).openAsync().thenCompose(store -> {
            for (Index index : common.getTargetIndexes()) {
                IndexState indexState = store.getIndexState(index);
                if (indexState != expectedIndexState) {
                    throw new RecordCoreStorageException("Unexpected index state", LogMessageKeys.INDEX_NAME, index.getName(), common.getRecordStoreBuilder().getSubspaceProvider().logKey(), common.getRecordStoreBuilder().getSubspaceProvider().toString(context), LogMessageKeys.INDEX_STATE, indexState, LogMessageKeys.INDEX_STATE_PRECONDITION, expectedIndexState);
                }
            }
            return function.apply(store);
        }), handlePostTransaction, onlineIndexerLogMessageKeyValues).handle((value, e) -> {
            if (e == null) {
                ret.complete(value);
                return AsyncUtil.READY_FALSE;
            } else {
                int currTries = tries.getAndIncrement();
                FDBException fdbE = getFDBException(e);
                if (currTries < common.config.getMaxRetries() && fdbE != null && lessenWorkCodes.contains(fdbE.getCode())) {
                    if (handleLessenWork != null) {
                        handleLessenWork.accept(fdbE, onlineIndexerLogMessageKeyValues);
                    }
                    long delay = (long) (Math.random() * toWait.get());
                    toWait.set(Math.min(toWait.get() * 2, common.getRunner().getDatabase().getFactory().getMaxDelayMillis()));
                    if (LOGGER.isWarnEnabled()) {
                        final KeyValueLogMessage message = KeyValueLogMessage.build("Retrying Runner Exception", LogMessageKeys.INDEXER_CURR_RETRY, currTries, LogMessageKeys.INDEXER_MAX_RETRIES, common.config.getMaxRetries(), LogMessageKeys.DELAY, delay, LogMessageKeys.LIMIT, limit);
                        message.addKeysAndValues(onlineIndexerLogMessageKeyValues);
                        LOGGER.warn(message.toString(), e);
                    }
                    return MoreAsyncUtil.delayedFuture(delay, TimeUnit.MILLISECONDS).thenApply(vignore3 -> true);
                } else {
                    return completeExceptionally(ret, e, onlineIndexerLogMessageKeyValues);
                }
            }
        }).thenCompose(Function.identity());
    }, common.getRunner().getExecutor()).whenComplete((vignore, e) -> {
        if (e != null) {
            // Just update ret and ignore the returned future.
            completeExceptionally(ret, e, onlineIndexerLogMessageKeyValues);
        }
    });
    return ret;
}
Also used : Arrays(java.util.Arrays) LogMessageKeys(com.apple.foundationdb.record.logging.LogMessageKeys) BiFunction(java.util.function.BiFunction) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Function(java.util.function.Function) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) Pair(org.apache.commons.lang3.tuple.Pair) FDBError(com.apple.foundationdb.FDBError) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LoggableException(com.apple.foundationdb.util.LoggableException) BiConsumer(java.util.function.BiConsumer) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) Logger(org.slf4j.Logger) IndexState(com.apple.foundationdb.record.IndexState) Set(java.util.Set) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Index(com.apple.foundationdb.record.metadata.Index) FDBException(com.apple.foundationdb.FDBException) API(com.apple.foundationdb.annotation.API) ArrayList(java.util.ArrayList) FDBException(com.apple.foundationdb.FDBException) Index(com.apple.foundationdb.record.metadata.Index) IndexState(com.apple.foundationdb.record.IndexState) KeyValueLogMessage(com.apple.foundationdb.record.logging.KeyValueLogMessage) RecordCoreStorageException(com.apple.foundationdb.record.RecordCoreStorageException) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicLong(java.util.concurrent.atomic.AtomicLong) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Nonnull(javax.annotation.Nonnull)

Example 2 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class BunchedMapScanTest method scanWithConflict.

@Test
public void scanWithConflict() throws InterruptedException, ExecutionException {
    clearAndPopulate();
    try (Transaction tr1 = db.createTransaction();
        Transaction tr2 = db.createTransaction()) {
        CompletableFuture.allOf(tr1.getReadVersion(), tr2.getReadVersion()).get();
        BunchedMapIterator<Tuple, Tuple> iterator = map.scan(tr1, bmSubspace);
        int count = MoreAsyncUtil.reduce(iterator, 0, (oldCount, item) -> oldCount + 1).get();
        assertEquals(keys.size(), count);
        tr1.addWriteConflictKey(Tuple.from(count).pack());
        assertFalse(map.put(tr2, bmSubspace, Tuple.from(keys.get(keys.size() - 1).getLong(0) + 1), value).get().isPresent());
        tr2.commit().get();
        CompletionException e = assertThrows(CompletionException.class, () -> tr1.commit().join());
        assertNotNull(e.getCause());
        assertTrue(e.getCause() instanceof FDBException);
        FDBException fdbE = (FDBException) e.getCause();
        assertEquals(FDBError.NOT_COMMITTED.code(), fdbE.getCode());
    }
    byte[] continuation = null;
    for (int i = 0; i < keys.size(); i++) {
    }
    try (Transaction tr1 = db.createTransaction();
        Transaction tr2 = db.createTransaction()) {
        CompletableFuture.allOf(tr1.getReadVersion(), tr2.getReadVersion()).get();
        BunchedMapIterator<Tuple, Tuple> iterator = map.scan(tr1, bmSubspace);
    }
}
Also used : IntStream(java.util.stream.IntStream) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) FDB(com.apple.foundationdb.FDB) Arrays(java.util.Arrays) FDBTestBase(com.apple.foundationdb.FDBTestBase) BiFunction(java.util.function.BiFunction) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) CompletableFuture(java.util.concurrent.CompletableFuture) Database(com.apple.foundationdb.Database) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) Supplier(java.util.function.Supplier) Subspace(com.apple.foundationdb.subspace.Subspace) ArrayList(java.util.ArrayList) AfterAll(org.junit.jupiter.api.AfterAll) Transaction(com.apple.foundationdb.Transaction) Tuple(com.apple.foundationdb.tuple.Tuple) Lists(com.google.common.collect.Lists) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) FDBError(com.apple.foundationdb.FDBError) BeforeAll(org.junit.jupiter.api.BeforeAll) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) NoSuchElementException(java.util.NoSuchElementException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ByteArrayUtil2(com.apple.foundationdb.tuple.ByteArrayUtil2) MoreAsyncUtil(com.apple.foundationdb.async.MoreAsyncUtil) ByteArrayUtil(com.apple.foundationdb.tuple.ByteArrayUtil) LongStream(java.util.stream.LongStream) Tags(com.apple.test.Tags) DirectoryLayer(com.apple.foundationdb.directory.DirectoryLayer) CompletionException(java.util.concurrent.CompletionException) Collectors(java.util.stream.Collectors) ReadTransaction(com.apple.foundationdb.ReadTransaction) Test(org.junit.jupiter.api.Test) ExecutionException(java.util.concurrent.ExecutionException) List(java.util.List) PathUtil(com.apple.foundationdb.directory.PathUtil) Stream(java.util.stream.Stream) FDBException(com.apple.foundationdb.FDBException) TupleHelpers(com.apple.foundationdb.tuple.TupleHelpers) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Comparator(java.util.Comparator) Collections(java.util.Collections) AsyncPeekIterator(com.apple.foundationdb.async.AsyncPeekIterator) Transaction(com.apple.foundationdb.Transaction) ReadTransaction(com.apple.foundationdb.ReadTransaction) CompletionException(java.util.concurrent.CompletionException) FDBException(com.apple.foundationdb.FDBException) Tuple(com.apple.foundationdb.tuple.Tuple) Test(org.junit.jupiter.api.Test)

Example 3 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method inconsistentScan.

private static List<KeyValue> inconsistentScan(@Nonnull Database db, @Nonnull Subspace subspace) {
    // Note that tr is mutated in the block, hence not using try-with-resources
    Transaction tr = db.createTransaction();
    try {
        KeySelector begin = KeySelector.firstGreaterOrEqual(subspace.range().begin);
        KeySelector end = KeySelector.firstGreaterOrEqual(subspace.range().end);
        KeyValue lastSeen = null;
        AsyncIterator<KeyValue> rangeIterator = tr.getRange(begin, end).iterator();
        List<KeyValue> rangeKVs = new ArrayList<>();
        boolean done = false;
        while (!done) {
            // Might loop if there are timeouts encountered within loop.
            try {
                while (rangeIterator.hasNext()) {
                    KeyValue next = rangeIterator.next();
                    rangeKVs.add(next);
                    lastSeen = next;
                }
                done = true;
            } catch (RuntimeException e) {
                FDBException fdbE = unwrapException(e);
                if (fdbE == null || fdbE.getCode() != FDBError.TRANSACTION_TOO_OLD.code()) {
                    throw e;
                } else {
                    // Timed out. Restart transaction and keep going.
                    tr.close();
                    tr = db.createTransaction();
                    if (lastSeen != null) {
                        // Update begin if we have any results.
                        begin = KeySelector.firstGreaterThan(lastSeen.getKey());
                        lastSeen = null;
                    }
                    rangeIterator = tr.getRange(begin, end).iterator();
                }
            }
        }
        return rangeKVs;
    } finally {
        tr.close();
    }
}
Also used : KeyValue(com.apple.foundationdb.KeyValue) Transaction(com.apple.foundationdb.Transaction) ArrayList(java.util.ArrayList) FDBException(com.apple.foundationdb.FDBException) KeySelector(com.apple.foundationdb.KeySelector)

Example 4 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class BunchedMapTest method runWithTwoTrs.

private void runWithTwoTrs(@Nonnull BiConsumer<? super Transaction, ? super Transaction> operation, boolean legal, @Nonnull List<Tuple> boundaryKeys) throws ExecutionException, InterruptedException {
    final String id = "two-trs-" + UUID.randomUUID().toString();
    try (Transaction tr1 = db.createTransaction();
        Transaction tr2 = db.createTransaction()) {
        tr1.options().setDebugTransactionIdentifier(id + "-1");
        tr1.options().setLogTransaction();
        tr2.options().setDebugTransactionIdentifier(id + "-2");
        tr2.options().setLogTransaction();
        CompletableFuture.allOf(tr1.getReadVersion(), tr2.getReadVersion()).get();
        tr1.addWriteConflictKey(new byte[] { 0x01 });
        tr2.addWriteConflictKey(new byte[] { 0x02 });
        operation.accept(tr1, tr2);
        tr1.commit().get();
        if (legal) {
            tr2.commit().get();
        } else {
            ExecutionException e = assertThrows(ExecutionException.class, () -> tr2.commit().get());
            assertNotNull(e.getCause());
            assertTrue(e.getCause() instanceof FDBException);
            FDBException fdbE = (FDBException) e.getCause();
            assertEquals(FDBError.NOT_COMMITTED.code(), fdbE.getCode());
        }
    }
    verifyBoundaryKeys(boundaryKeys);
}
Also used : Transaction(com.apple.foundationdb.Transaction) FDBException(com.apple.foundationdb.FDBException) ExecutionException(java.util.concurrent.ExecutionException)

Example 5 with FDBException

use of com.apple.foundationdb.FDBException in project fdb-record-layer by FoundationDB.

the class FDBDatabaseRunnerTest method testRestoreMdc.

@Test
void testRestoreMdc() {
    Executor oldExecutor = FDBDatabaseFactory.instance().getExecutor();
    try {
        ThreadContext.clearAll();
        ThreadContext.put("outer", "Echidna");
        final Map<String, String> outer = ThreadContext.getContext();
        final ImmutableMap<String, String> restored = ImmutableMap.of("restored", "Platypus");
        FDBDatabaseFactory.instance().setExecutor(new ContextRestoringExecutor(new ForkJoinPool(2), ImmutableMap.of("executor", "Water Bear")));
        AtomicInteger attempts = new AtomicInteger(0);
        final FDBDatabaseRunner runner = database.newRunner(FDBRecordContextConfig.newBuilder().setMdcContext(restored));
        List<Map<String, String>> threadContexts = new Vector<>();
        Consumer<String> saveThreadContext = name -> threadContexts.add(threadContextPlus(name, attempts.get(), ThreadContext.getContext()));
        final String runnerRunAsyncName = "runner runAsync";
        final String supplyAsyncName = "supplyAsync";
        final String handleName = "handle";
        // Delay starting the future until all callbacks have been set up so that the handle lambda
        // runs in the context-restoring executor.
        CompletableFuture<Void> signal = new CompletableFuture<>();
        CompletableFuture<?> task = runner.runAsync(recordContext -> {
            saveThreadContext.accept(runnerRunAsyncName);
            return signal.thenCompose(vignore -> CompletableFuture.supplyAsync(() -> {
                saveThreadContext.accept(supplyAsyncName);
                if (attempts.getAndIncrement() == 0) {
                    throw new RecordCoreRetriableTransactionException("Retriable and lessener", new FDBException("not_committed", 1020));
                } else {
                    return null;
                }
            }, recordContext.getExecutor()));
        }).handle((result, exception) -> {
            saveThreadContext.accept(handleName);
            return exception;
        });
        signal.complete(null);
        assertNull(task.join());
        List<Map<String, String>> expected = ImmutableList.of(// it is known behavior that the first will be run in the current context
        threadContextPlus(runnerRunAsyncName, 0, outer), threadContextPlus(supplyAsyncName, 0, restored), // the code that creates the future, should now have the correct MDC
        threadContextPlus(runnerRunAsyncName, 1, restored), threadContextPlus(supplyAsyncName, 1, restored), // this should also have the correct MDC
        threadContextPlus(handleName, 2, restored));
        assertEquals(expected, threadContexts);
        assertEquals(outer, ThreadContext.getContext());
    } finally {
        FDBDatabaseFactory.instance().setExecutor(oldExecutor);
    }
}
Also used : Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) RecordMetaData(com.apple.foundationdb.record.RecordMetaData) BeforeEach(org.junit.jupiter.api.BeforeEach) FDBDatabaseTest.testStoreAndRetrieveSimpleRecord(com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseTest.testStoreAndRetrieveSimpleRecord) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) CompletableFuture(java.util.concurrent.CompletableFuture) AsyncUtil(com.apple.foundationdb.async.AsyncUtil) AtomicReference(java.util.concurrent.atomic.AtomicReference) Tuple(com.apple.foundationdb.tuple.Tuple) Vector(java.util.Vector) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) ImmutableList(com.google.common.collect.ImmutableList) FDBError(com.apple.foundationdb.FDBError) RecordCoreException(com.apple.foundationdb.record.RecordCoreException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ThreadContext(org.apache.logging.log4j.ThreadContext) TestRecords1Proto(com.apple.foundationdb.record.TestRecords1Proto) ImmutableMap(com.google.common.collect.ImmutableMap) Executor(java.util.concurrent.Executor) Tags(com.apple.test.Tags) CompletionException(java.util.concurrent.CompletionException) UUID(java.util.UUID) Test(org.junit.jupiter.api.Test) Consumer(java.util.function.Consumer) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) FDBException(com.apple.foundationdb.FDBException) ForkJoinPool(java.util.concurrent.ForkJoinPool) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Message(com.google.protobuf.Message) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) FDBException(com.apple.foundationdb.FDBException) CompletableFuture(java.util.concurrent.CompletableFuture) Executor(java.util.concurrent.Executor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordCoreRetriableTransactionException(com.apple.foundationdb.record.RecordCoreRetriableTransactionException) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) Vector(java.util.Vector) ForkJoinPool(java.util.concurrent.ForkJoinPool) Test(org.junit.jupiter.api.Test)

Aggregations

FDBException (com.apple.foundationdb.FDBException)28 Test (org.junit.jupiter.api.Test)25 FDBError (com.apple.foundationdb.FDBError)16 CompletableFuture (java.util.concurrent.CompletableFuture)16 List (java.util.List)15 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)15 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)15 AsyncUtil (com.apple.foundationdb.async.AsyncUtil)14 Assertions.assertTrue (org.junit.jupiter.api.Assertions.assertTrue)14 RecordCoreException (com.apple.foundationdb.record.RecordCoreException)13 Assertions.assertNotNull (org.junit.jupiter.api.Assertions.assertNotNull)13 Assertions.assertNull (org.junit.jupiter.api.Assertions.assertNull)13 Assertions.assertThrows (org.junit.jupiter.api.Assertions.assertThrows)13 TestRecords1Proto (com.apple.foundationdb.record.TestRecords1Proto)12 Tuple (com.apple.foundationdb.tuple.Tuple)12 RecordCoreRetriableTransactionException (com.apple.foundationdb.record.RecordCoreRetriableTransactionException)11 Assertions.fail (org.junit.jupiter.api.Assertions.fail)11 Tags (com.apple.test.Tags)10 Message (com.google.protobuf.Message)10 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)10