Examples with CircuitBreaker org.opensearch.common.breaker.CircuitBreaker used on opensource projects

Example 1 with CircuitBreaker

use of org.opensearch.common.breaker.CircuitBreaker in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerService method memoryUsed.

private MemoryUsage memoryUsed(long newBytesReserved) {
    long transientUsage = 0;
    long permanentUsage = 0;
    for (CircuitBreaker breaker : this.breakers.values()) {
        long breakerUsed = (long) (breaker.getUsed() * breaker.getOverhead());
        if (breaker.getDurability() == CircuitBreaker.Durability.TRANSIENT) {
            transientUsage += breakerUsed;
        } else if (breaker.getDurability() == CircuitBreaker.Durability.PERMANENT) {
            permanentUsage += breakerUsed;
        }
    }
    if (this.trackRealMemoryUsage) {
        final long current = currentMemoryUsage();
        return new MemoryUsage(current, current + newBytesReserved, transientUsage, permanentUsage);
    } else {
        long parentEstimated = transientUsage + permanentUsage;
        return new MemoryUsage(parentEstimated, parentEstimated, transientUsage, permanentUsage);
    }
}

Example 2 with CircuitBreaker

use of org.opensearch.common.breaker.CircuitBreaker in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerServiceTests method testAllocationBucketsBreaker.

public void testAllocationBucketsBreaker() {
    Settings clusterSettings = Settings.builder().put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "100b").put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), "false").build();
    try (HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, Collections.emptyList(), new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
        long parentLimitBytes = service.getParentLimit();
        assertEquals(new ByteSizeValue(100, ByteSizeUnit.BYTES).getBytes(), parentLimitBytes);
        CircuitBreaker breaker = service.getBreaker(CircuitBreaker.REQUEST);
        MultiBucketConsumerService.MultiBucketConsumer multiBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer(10000, breaker);
        // make sure used bytes is greater than the total circuit breaker limit
        breaker.addWithoutBreaking(200);
        // make sure that we check on the the following call
        for (int i = 0; i < 1023; i++) {
            multiBucketConsumer.accept(0);
        }
        CircuitBreakingException exception = expectThrows(CircuitBreakingException.class, () -> multiBucketConsumer.accept(1024));
        assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [allocated_buckets] would be"));
        assertThat(exception.getMessage(), containsString("which is larger than the limit of [100/100b]"));
    }
}

Example 3 with CircuitBreaker

use of org.opensearch.common.breaker.CircuitBreaker in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerServiceTests method testBorrowingSiblingBreakerMemory.

/**
 * Test that a breaker correctly redistributes to a different breaker, in
 * this case, the request breaker borrows space from the fielddata breaker
 */
public void testBorrowingSiblingBreakerMemory() {
    Settings clusterSettings = Settings.builder().put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false).put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "200mb").put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb").build();
    try (CircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, Collections.emptyList(), new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
        CircuitBreaker requestCircuitBreaker = service.getBreaker(CircuitBreaker.REQUEST);
        CircuitBreaker fieldDataCircuitBreaker = service.getBreaker(CircuitBreaker.FIELDDATA);
        assertEquals(new ByteSizeValue(200, ByteSizeUnit.MB).getBytes(), service.stats().getStats(CircuitBreaker.PARENT).getLimit());
        assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getLimit());
        assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), fieldDataCircuitBreaker.getLimit());
        double fieldDataUsedBytes = fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break");
        assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), fieldDataUsedBytes, 0.0);
        double requestUsedBytes = requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break");
        assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), requestUsedBytes, 0.0);
        requestUsedBytes = requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break");
        assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).getBytes(), requestUsedBytes, 0.0);
        CircuitBreakingException exception = expectThrows(CircuitBreakingException.class, () -> requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should break"));
        assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be"));
        assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]"));
        assertThat(exception.getMessage(), containsString("usages [request=157286400/150mb, fielddata=54001664/51.5mb, in_flight_requests=0/0b]"));
        assertThat(exception.getDurability(), equalTo(CircuitBreaker.Durability.TRANSIENT));
    }
}

Example 4 with CircuitBreaker

use of org.opensearch.common.breaker.CircuitBreaker in project OpenSearch by opensearch-project.

the class InboundPipelineTests method testDecodeExceptionIsPropagated.

public void testDecodeExceptionIsPropagated() throws IOException {
    BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
    };
    final StatsTracker statsTracker = new StatsTracker();
    final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
    final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
    final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
    final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
    try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
        String actionName = "actionName";
        final Version invalidVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
        final String value = randomAlphaOfLength(1000);
        final boolean isRequest = randomBoolean();
        final long requestId = randomNonNegativeLong();
        OutboundMessage message;
        if (isRequest) {
            message = new OutboundMessage.Request(threadContext, new String[0], new TestRequest(value), invalidVersion, actionName, requestId, false, false);
        } else {
            message = new OutboundMessage.Response(threadContext, Collections.emptySet(), new TestResponse(value), invalidVersion, requestId, false, false);
        }
        final BytesReference reference = message.serialize(streamOutput);
        try (ReleasableBytesReference releasable = ReleasableBytesReference.wrap(reference)) {
            expectThrows(IllegalStateException.class, () -> pipeline.handleBytes(new FakeTcpChannel(), releasable));
        }
        // Pipeline cannot be reused after uncaught exception
        final IllegalStateException ise = expectThrows(IllegalStateException.class, () -> pipeline.handleBytes(new FakeTcpChannel(), ReleasableBytesReference.wrap(BytesArray.EMPTY)));
        assertEquals("Pipeline state corrupted by uncaught exception", ise.getMessage());
    }
}

Example 5 with CircuitBreaker

use of org.opensearch.common.breaker.CircuitBreaker in project OpenSearch by opensearch-project.

the class InboundPipelineTests method testEnsureBodyIsNotPrematurelyReleased.

public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
    BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
    };
    final StatsTracker statsTracker = new StatsTracker();
    final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
    final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
    final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
    final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
    try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
        String actionName = "actionName";
        final Version version = Version.CURRENT;
        final String value = randomAlphaOfLength(1000);
        final boolean isRequest = randomBoolean();
        final long requestId = randomNonNegativeLong();
        OutboundMessage message;
        if (isRequest) {
            message = new OutboundMessage.Request(threadContext, new String[0], new TestRequest(value), version, actionName, requestId, false, false);
        } else {
            message = new OutboundMessage.Response(threadContext, Collections.emptySet(), new TestResponse(value), version, requestId, false, false);
        }
        final BytesReference reference = message.serialize(streamOutput);
        final int fixedHeaderSize = TcpHeader.headerSize(Version.CURRENT);
        final int variableHeaderSize = reference.getInt(fixedHeaderSize - 4);
        final int totalHeaderSize = fixedHeaderSize + variableHeaderSize;
        final AtomicBoolean bodyReleased = new AtomicBoolean(false);
        for (int i = 0; i < totalHeaderSize - 1; ++i) {
            try (ReleasableBytesReference slice = ReleasableBytesReference.wrap(reference.slice(i, 1))) {
                pipeline.handleBytes(new FakeTcpChannel(), slice);
            }
        }
        final Releasable releasable = () -> bodyReleased.set(true);
        final int from = totalHeaderSize - 1;
        final BytesReference partHeaderPartBody = reference.slice(from, reference.length() - from - 1);
        try (ReleasableBytesReference slice = new ReleasableBytesReference(partHeaderPartBody, releasable)) {
            pipeline.handleBytes(new FakeTcpChannel(), slice);
        }
        assertFalse(bodyReleased.get());
        try (ReleasableBytesReference slice = new ReleasableBytesReference(reference.slice(reference.length() - 1, 1), releasable)) {
            pipeline.handleBytes(new FakeTcpChannel(), slice);
        }
        assertTrue(bodyReleased.get());
    }
}