Examples with CircuitBreakerState fish.payara.microprofile.faulttolerance.state.CircuitBreakerState used on opensource projects

Example 1 with CircuitBreakerState

use of fish.payara.microprofile.faulttolerance.state.CircuitBreakerState in project Payara by payara.

the class FaultToleranceService method registerCircuitBreaker.

/**
 * Helper method to create and register a CircuitBreakerState object for an annotated method
 * @param applicationName The application name to register the CircuitBreakerState against
 * @param fullMethodSignature The method signature to register the CircuitBreakerState against
 * @param bulkheadWaitingTaskQueue The CircuitBreaker annotation of the annotated method
 * @return The CircuitBreakerState object for the given method signature and application
 */
private synchronized CircuitBreakerState registerCircuitBreaker(String applicationName, String fullMethodSignature, CircuitBreaker circuitBreaker) {
    // Double lock as multiple methods can get inside the calling if at the same time
    logger.log(Level.FINER, "Checking double lock to see if something else has already added the application " + "to the circuit breaker states map");
    if (circuitBreakerStates.get(applicationName) == null) {
        logger.log(Level.FINER, "Registering application to the circuit breaker states map: {0}", applicationName);
        circuitBreakerStates.put(applicationName, new ConcurrentHashMap<>());
    }
    // Double lock as multiple methods can get inside the calling if at the same time
    logger.log(Level.FINER, "Checking double lock to see if something else has already added the annotated method " + "to the circuit breaker states map");
    if (circuitBreakerStates.get(applicationName).get(fullMethodSignature) == null) {
        logger.log(Level.FINER, "Registering CircuitBreakerState for method {0} to the circuit breaker states map " + "for application {1}", new String[] { fullMethodSignature, applicationName });
        circuitBreakerStates.get(applicationName).put(fullMethodSignature, new CircuitBreakerState(circuitBreaker.requestVolumeThreshold()));
    }
    return circuitBreakerStates.get(applicationName).get(fullMethodSignature);
}

Example 2 with CircuitBreakerState

use of fish.payara.microprofile.faulttolerance.state.CircuitBreakerState in project Payara by payara.

the class FaultToleranceMetrics method boundTo.

/**
 * @return a instance on the basis of this {@link FaultToleranceMetrics} that is properly adopted to the provided
 *         context and policy. This can be the same instance if no change is required or a new one if a change of
 *         internal state is required.
 *
 *         As part of binding to the context and policy this method also should register all metrics that make sense
 *         for the provided policy in case this has not been done already.
 */
default FaultToleranceMetrics boundTo(FaultToleranceMethodContext context, FaultTolerancePolicy policy) {
    if (policy.isMetricsEnabled) {
        String[] fallbackTag = policy.isFallbackPresent() ? new String[] { "fallback", "applied", "notApplied" } : new String[] { "fallback", "notDefined" };
        register(MetricType.COUNTER, "ft.invocations.total", new String[][] { { "result", "valueReturned", "exceptionThrown" }, fallbackTag });
        if (policy.isRetryPresent()) {
            List<String> retryResultTag = new ArrayList<>(asList("retryResult", "valueReturned", "exceptionNotRetryable"));
            if (policy.retry.isMaxRetriesSet()) {
                retryResultTag.add("maxRetriesReached");
            }
            if (policy.retry.isMaxDurationSet()) {
                retryResultTag.add("maxDurationReached");
            }
            register(MetricType.COUNTER, "ft.retry.calls.total", new String[][] { { "retried", "true", "false" }, retryResultTag.toArray(new String[0]) });
            register(MetricType.COUNTER, "ft.retry.retries.total");
        }
        if (policy.isTimeoutPresent()) {
            register(MetricType.COUNTER, "ft.timeout.calls.total", new String[][] { { "timedOut", "true", "false" } });
            register(MetricType.HISTOGRAM, "ft.timeout.executionDuration");
        }
        if (policy.isCircuitBreakerPresent()) {
            register(MetricType.COUNTER, "ft.circuitbreaker.calls.total", new String[][] { { "circuitBreakerResult", "success", "failure", "circuitBreakerOpen" } });
            CircuitBreakerState state = context.getState();
            register("ft.circuitbreaker.state.total", MetricUnits.NANOSECONDS, state::nanosOpen, "state", "open");
            register("ft.circuitbreaker.state.total", MetricUnits.NANOSECONDS, state::nanosHalfOpen, "state", "halfOpen");
            register("ft.circuitbreaker.state.total", MetricUnits.NANOSECONDS, state::nanosClosed, "state", "closed");
            register(MetricType.COUNTER, "ft.circuitbreaker.opened.total");
        }
        if (policy.isBulkheadPresent()) {
            register(MetricType.COUNTER, "ft.bulkhead.calls.total", new String[][] { { "bulkheadResult", "accepted", "rejected" } });
            register(MetricType.HISTOGRAM, "ft.bulkhead.runningDuration");
            if (policy.isAsynchronous()) {
                BlockingQueue<Thread> running = context.getConcurrentExecutions();
                register("ft.bulkhead.executionsRunning", null, running::size);
                AtomicInteger queuingOrRunning = context.getQueuingOrRunningPopulation();
                register("ft.bulkhead.executionsWaiting", null, () -> Math.max(0, queuingOrRunning.get() - policy.bulkhead.value));
                register(MetricType.HISTOGRAM, "ft.bulkhead.waitingDuration");
            } else {
                AtomicInteger running = context.getQueuingOrRunningPopulation();
                register("ft.bulkhead.executionsRunning", null, running::get);
            }
        }
    }
    return this;
}

Example 3 with CircuitBreakerState

use of fish.payara.microprofile.faulttolerance.state.CircuitBreakerState in project Payara by payara.

the class FaultTolerancePolicy method processCircuitBreakerStage.

/**
 * Stage that takes care of the {@link CircuitBreakerPolicy} handling.
 */
private Object processCircuitBreakerStage(FaultToleranceInvocation invocation, AsyncFuture asyncAttempt) throws Exception {
    if (!isCircuitBreakerPresent()) {
        return processTimeoutStage(invocation, asyncAttempt);
    }
    logger.log(Level.FINER, "Proceeding invocation with circuitbreaker semantics");
    CircuitBreakerState state = invocation.context.getState();
    Object resultValue = null;
    switch(state.getCircuitState()) {
        default:
        case OPEN:
            logger.log(Level.FINER, "CircuitBreaker is open, throwing exception");
            invocation.metrics.incrementCircuitbreakerCallsPreventedTotal();
            throw new CircuitBreakerOpenException();
        case HALF_OPEN:
            logger.log(Level.FINER, "Proceeding half open CircuitBreaker context");
            try {
                resultValue = processTimeoutStage(invocation, asyncAttempt);
            } catch (Exception | Error ex) {
                if (circuitBreaker.isFailure(ex)) {
                    invocation.metrics.incrementCircuitbreakerCallsFailedTotal();
                    logger.log(Level.FINE, "Exception causes CircuitBreaker to transit: half-open => open");
                    openCircuit(invocation, state);
                } else {
                    invocation.metrics.incrementCircuitbreakerCallsSucceededTotal();
                }
                throw ex;
            }
            if (state.halfOpenSuccessfulClosedCircuit(circuitBreaker.successThreshold)) {
                logger.log(Level.FINE, "Success threshold causes CircuitBreaker to transit: half-open => closed");
            }
            invocation.metrics.incrementCircuitbreakerCallsSucceededTotal();
            return resultValue;
        case CLOSED:
            logger.log(Level.FINER, "Proceeding closed CircuitBreaker context");
            Throwable failedOn = null;
            try {
                resultValue = processTimeoutStage(invocation, asyncAttempt);
                state.recordClosedOutcome(true);
            } catch (Exception | Error ex) {
                if (circuitBreaker.isFailure(ex)) {
                    state.recordClosedOutcome(false);
                    invocation.metrics.incrementCircuitbreakerCallsFailedTotal();
                } else {
                    state.recordClosedOutcome(true);
                    invocation.metrics.incrementCircuitbreakerCallsSucceededTotal();
                }
                failedOn = ex;
            }
            if (state.isOverFailureThreshold()) {
                logger.log(Level.FINE, "Failure threshold causes CircuitBreaker to transit: closed => open");
                openCircuit(invocation, state);
            }
            if (failedOn != null) {
                rethrow(failedOn);
            }
            invocation.metrics.incrementCircuitbreakerCallsSucceededTotal();
            return resultValue;
    }
}

Example 4 with CircuitBreakerState

use of fish.payara.microprofile.faulttolerance.state.CircuitBreakerState in project Payara by payara.

the class CircuitBreakerInterceptor method circuitBreak.

private Object circuitBreak(InvocationContext invocationContext) throws Exception {
    Object proceededInvocationContext = null;
    FaultToleranceService faultToleranceService = Globals.getDefaultBaseServiceLocator().getService(FaultToleranceService.class);
    CircuitBreaker circuitBreaker = FaultToleranceCdiUtils.getAnnotation(beanManager, CircuitBreaker.class, invocationContext);
    Config config = null;
    try {
        config = ConfigProvider.getConfig();
    } catch (IllegalArgumentException ex) {
        logger.log(Level.INFO, "No config could be found", ex);
    }
    Class<? extends Throwable>[] failOn = circuitBreaker.failOn();
    try {
        String failOnString = ((String) FaultToleranceCdiUtils.getOverrideValue(config, Retry.class, "failOn", invocationContext, String.class).get());
        List<Class> classList = new ArrayList<>();
        // Remove any curly or square brackets from the string, as well as any spaces and ".class"es and loop
        for (String className : failOnString.replaceAll("[\\{\\[ \\]\\}]", "").replaceAll("\\.class", "").split(",")) {
            // Get a class object
            classList.add(Class.forName(className));
        }
        failOn = classList.toArray(failOn);
    } catch (NoSuchElementException nsee) {
        logger.log(Level.FINER, "Could not find element in config", nsee);
    } catch (ClassNotFoundException cnfe) {
        logger.log(Level.INFO, "Could not find class from failOn config, defaulting to annotation. " + "Make sure you give the full canonical class name.", cnfe);
    }
    long delay = (Long) FaultToleranceCdiUtils.getOverrideValue(config, CircuitBreaker.class, "value", invocationContext, Long.class).orElse(circuitBreaker.delay());
    ChronoUnit delayUnit = (ChronoUnit) FaultToleranceCdiUtils.getOverrideValue(config, CircuitBreaker.class, "delayUnit", invocationContext, ChronoUnit.class).orElse(circuitBreaker.delayUnit());
    int requestVolumeThreshold = (Integer) FaultToleranceCdiUtils.getOverrideValue(config, CircuitBreaker.class, "requestVolumeThreshold", invocationContext, Integer.class).orElse(circuitBreaker.requestVolumeThreshold());
    double failureRatio = (Double) FaultToleranceCdiUtils.getOverrideValue(config, CircuitBreaker.class, "failureRatio", invocationContext, Double.class).orElse(circuitBreaker.failureRatio());
    int successThreshold = (Integer) FaultToleranceCdiUtils.getOverrideValue(config, CircuitBreaker.class, "successThreshold", invocationContext, Integer.class).orElse(circuitBreaker.successThreshold());
    long delayMillis = Duration.of(delay, delayUnit).toMillis();
    InvocationManager invocationManager = Globals.getDefaultBaseServiceLocator().getService(InvocationManager.class);
    CircuitBreakerState circuitBreakerState = faultToleranceService.getCircuitBreakerState(faultToleranceService.getApplicationName(invocationManager, invocationContext), invocationContext.getMethod(), circuitBreaker);
    switch(circuitBreakerState.getCircuitState()) {
        case OPEN:
            logger.log(Level.FINER, "CircuitBreaker is Open, throwing exception");
            // If open, immediately throw an error
            throw new CircuitBreakerOpenException("CircuitBreaker for method " + invocationContext.getMethod().getName() + "is in state OPEN.");
        case CLOSED:
            // If closed, attempt to proceed the invocation context
            try {
                logger.log(Level.FINER, "Proceeding CircuitBreaker context");
                proceededInvocationContext = invocationContext.proceed();
            } catch (Exception ex) {
                logger.log(Level.FINE, "Exception executing CircuitBreaker context");
                // Check if the exception is something that should record a failure
                if (shouldFail(failOn, ex)) {
                    logger.log(Level.FINE, "Caught exception is included in CircuitBreaker failOn, " + "recording failure against CircuitBreaker");
                    // Add a failure result to the queue
                    circuitBreakerState.recordClosedResult(Boolean.FALSE);
                    // Calculate the failure threshold
                    long failureThreshold = Math.round(requestVolumeThreshold * failureRatio);
                    // If we're over the failure threshold, open the circuit
                    if (circuitBreakerState.isOverFailureThreshold(failureThreshold)) {
                        logger.log(Level.FINE, "CircuitBreaker is over failure threshold {0}, opening circuit", failureThreshold);
                        circuitBreakerState.setCircuitState(CircuitBreakerState.CircuitState.OPEN);
                        // Kick off a thread that will half-open the circuit after the specified delay
                        scheduleHalfOpen(delayMillis, circuitBreakerState);
                    }
                }
                // Finally, propagate the error upwards
                throw ex;
            }
            // If everything is bon, just add a success value
            circuitBreakerState.recordClosedResult(Boolean.TRUE);
            break;
        case HALF_OPEN:
            // If half-open, attempt to proceed the invocation context
            try {
                logger.log(Level.FINER, "Proceeding half open CircuitBreaker context");
                proceededInvocationContext = invocationContext.proceed();
            } catch (Exception ex) {
                logger.log(Level.FINE, "Exception executing CircuitBreaker context");
                // Check if the exception is something that should record a failure
                if (shouldFail(failOn, ex)) {
                    logger.log(Level.FINE, "Caught exception is included in CircuitBreaker failOn, " + "reopening half open circuit");
                    // Open the circuit again, and reset the half-open result counter
                    circuitBreakerState.setCircuitState(CircuitBreakerState.CircuitState.OPEN);
                    circuitBreakerState.resetHalfOpenSuccessfulResultCounter();
                    scheduleHalfOpen(delayMillis, circuitBreakerState);
                }
                throw ex;
            }
            // If the invocation context hasn't thrown an error, record a success
            circuitBreakerState.incrementHalfOpenSuccessfulResultCounter();
            logger.log(Level.FINER, "Number of consecutive successful circuitbreaker executions = {0}", circuitBreakerState.getHalfOpenSuccessFulResultCounter());
            // If we've hit the success threshold, close the circuit
            if (circuitBreakerState.getHalfOpenSuccessFulResultCounter() == successThreshold) {
                logger.log(Level.FINE, "Number of consecutive successful CircuitBreaker executions is above " + "threshold {0}, closing circuit", successThreshold);
                circuitBreakerState.setCircuitState(CircuitBreakerState.CircuitState.CLOSED);
                // Reset the counter for when we next need to use it
                circuitBreakerState.resetHalfOpenSuccessfulResultCounter();
                // Reset the rolling results window
                circuitBreakerState.resetResults();
            }
            break;
    }
    return proceededInvocationContext;
}

Example 5 with CircuitBreakerState

use of fish.payara.microprofile.faulttolerance.state.CircuitBreakerState in project Payara by payara.

the class FaultToleranceService method getCircuitBreakerState.

/**
 * Gets the CircuitBreakerState object for a given application name and method. If a CircuitBreakerState hasn't
 * been registered for the given application name and method, it will register the given CircuitBreaker.
 * @param applicationName The application name
 * @param annotatedMethod The method annotated with @CircuitBreaker
 * @param circuitBreaker The @CircuitBreaker annotation from the annotated method
 * @return The CircuitBreakerState for the given application and method
 */
public CircuitBreakerState getCircuitBreakerState(String applicationName, Method annotatedMethod, CircuitBreaker circuitBreaker) {
    CircuitBreakerState circuitBreakerState;
    String fullMethodSignature = getFullMethodSignature(annotatedMethod);
    Map<String, CircuitBreakerState> annotatedMethodCircuitBreakerStates = circuitBreakerStates.get(applicationName);
    // return the one already registered
    if (annotatedMethodCircuitBreakerStates == null) {
        logger.log(Level.FINER, "No matching application in the circuit breaker states map, registering...");
        circuitBreakerState = registerCircuitBreaker(applicationName, fullMethodSignature, circuitBreaker);
    } else {
        circuitBreakerState = annotatedMethodCircuitBreakerStates.get(fullMethodSignature);
        // return the one already registered
        if (circuitBreakerState == null) {
            logger.log(Level.FINER, "No matching method in the circuit breaker states map, registering...");
            circuitBreakerState = registerCircuitBreaker(applicationName, fullMethodSignature, circuitBreaker);
        }
    }
    return circuitBreakerState;
}