Search in sources :

Example 11 with TraceContext

use of com.google.gerrit.server.logging.TraceContext in project gerrit by GerritCodeReview.

the class RetryHelper method execute.

/**
 * Executes an action and records the number of attempts and the timeout as metrics.
 *
 * @param actionType the type of the action
 * @param action the action which should be executed and retried on failure
 * @param opts options for retrying the action on failure
 * @param exceptionPredicate predicate to control on which exception the action should be retried
 * @return the result of executing the action
 * @throws Exception any error or exception that made the action fail, callers are expected to
 *     catch and inspect this Throwable to decide carefully whether it should be re-thrown
 */
<T> T execute(String actionType, Action<T> action, Options opts, Predicate<Throwable> exceptionPredicate) throws Exception {
    MetricListener listener = new MetricListener();
    try (TraceContext traceContext = TraceContext.open()) {
        RetryerBuilder<T> retryerBuilder = createRetryerBuilder(actionType, opts, t -> {
            // retried (e.g. LockFailure). The retry has good chances to succeed.
            if (exceptionPredicate.test(t)) {
                return true;
            }
            String actionName = opts.actionName().orElse("N/A");
            // Exception hooks may identify additional exceptions for retry.
            if (exceptionHooks.stream().anyMatch(h -> h.shouldRetry(actionType, actionName, t))) {
                return true;
            }
            // of the failure. If a trace was already done there is no need to retry.
            if (retryWithTraceOnFailure && opts.retryWithTrace().isPresent() && opts.retryWithTrace().get().test(t)) {
                // skipped.
                if (exceptionHooks.stream().anyMatch(h -> h.skipRetryWithTrace(actionType, actionName, t))) {
                    return false;
                }
                String cause = formatCause(t);
                if (!TraceContext.isTracing()) {
                    String traceId = "retry-on-failure-" + new RequestId();
                    traceContext.addTag(RequestId.Type.TRACE_ID, traceId).forceLogging();
                    logger.atWarning().withCause(t).log("AutoRetry: %s failed, retry with tracing enabled (cause = %s)", actionName, cause);
                    opts.onAutoTrace().ifPresent(c -> c.accept(traceId));
                    metrics.autoRetryCount.increment(actionType, actionName, cause);
                    return true;
                }
                // A non-recoverable failure occurred. We retried the operation with tracing
                // enabled and it failed again. Log the failure so that admin can see if it
                // differs from the failure that triggered the retry.
                logger.atWarning().withCause(t).log("AutoRetry: auto-retry of %s has failed (cause = %s)", actionName, cause);
                metrics.failuresOnAutoRetryCount.increment(actionType, actionName, cause);
                return false;
            }
            return false;
        });
        retryerBuilder.withRetryListener(listener);
        return executeWithTimeoutCount(actionType, action, opts, retryerBuilder.build(), listener);
    } finally {
        if (listener.getAttemptCount() > 1) {
            logger.atWarning().log("%s was attempted %d times", actionType, listener.getAttemptCount());
            metrics.attemptCounts.incrementBy(actionType, opts.actionName().orElse("N/A"), listener.getOriginalCause().map(this::formatCause).orElse("_unknown"), listener.getAttemptCount() - 1);
        }
    }
}
Also used : RequestId(com.google.gerrit.server.logging.RequestId) TraceContext(com.google.gerrit.server.logging.TraceContext)

Example 12 with TraceContext

use of com.google.gerrit.server.logging.TraceContext in project gerrit by GerritCodeReview.

the class TraceIT method workQueueCopyLoggingContext.

@Test
public void workQueueCopyLoggingContext() throws Exception {
    assertThat(LoggingContext.getInstance().getTags().isEmpty()).isTrue();
    assertForceLogging(false);
    try (TraceContext traceContext = TraceContext.open().forceLogging().addTag("foo", "bar")) {
        Map<String, ? extends Set<Object>> tagMap = LoggingContext.getInstance().getTags().asMap();
        assertThat(tagMap.keySet()).containsExactly("foo");
        assertThat(tagMap.get("foo")).containsExactly("bar");
        assertForceLogging(true);
        workQueue.createQueue(1, "test-queue").submit(() -> {
            // Verify that the tags and force logging flag have been propagated to the new
            // thread.
            Map<String, ? extends Set<Object>> threadTagMap = LoggingContext.getInstance().getTags().asMap();
            expect.that(threadTagMap.keySet()).containsExactly("foo");
            expect.that(threadTagMap.get("foo")).containsExactly("bar");
            expect.that(LoggingContext.getInstance().shouldForceLogging(null, null, false)).isTrue();
        }).get();
        // Verify that tags and force logging flag in the outer thread are still set.
        tagMap = LoggingContext.getInstance().getTags().asMap();
        assertThat(tagMap.keySet()).containsExactly("foo");
        assertThat(tagMap.get("foo")).containsExactly("bar");
        assertForceLogging(true);
    }
    assertThat(LoggingContext.getInstance().getTags().isEmpty()).isTrue();
    assertForceLogging(false);
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) TraceContext(com.google.gerrit.server.logging.TraceContext) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Map(java.util.Map) AbstractDaemonTest(com.google.gerrit.acceptance.AbstractDaemonTest) Test(org.junit.Test)

Aggregations

TraceContext (com.google.gerrit.server.logging.TraceContext)12 AuthException (com.google.gerrit.extensions.restapi.AuthException)5 RequestInfo (com.google.gerrit.server.RequestInfo)4 RequestId (com.google.gerrit.server.logging.RequestId)4 ImmutableSet (com.google.common.collect.ImmutableSet)3 Optional (java.util.Optional)3 Joiner (com.google.common.base.Joiner)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 ImmutableList (com.google.common.collect.ImmutableList)2 ListMultimap (com.google.common.collect.ListMultimap)2 FluentLogger (com.google.common.flogger.FluentLogger)2 BadRequestException (com.google.gerrit.extensions.restapi.BadRequestException)2 IdString (com.google.gerrit.extensions.restapi.IdString)2 InvalidDeadlineException (com.google.gerrit.server.InvalidDeadlineException)2 RequestStateContext (com.google.gerrit.server.cancellation.RequestStateContext)2 PerformanceLogContext (com.google.gerrit.server.logging.PerformanceLogContext)2 PermissionBackend (com.google.gerrit.server.permissions.PermissionBackend)2 PermissionBackendException (com.google.gerrit.server.permissions.PermissionBackendException)2 ChangeData (com.google.gerrit.server.query.change.ChangeData)2 HashSet (java.util.HashSet)2