Examples with CircuitBreakingException org.elasticsearch.common.breaker.CircuitBreakingException used on opensource projects

Example 1 with CircuitBreakingException

use of org.elasticsearch.common.breaker.CircuitBreakingException in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testParentChecking.

/**
     * Test that a breaker correctly redistributes to a different breaker, in
     * this case, the fielddata breaker borrows space from the request breaker
     */
@TestLogging("_root:DEBUG,org.elasticsearch.action.search:TRACE")
public void testParentChecking() throws Exception {
    if (noopBreakerUsed()) {
        logger.info("--> noop breakers used, skipping test");
        return;
    }
    assertAcked(prepareCreate("cb-test", 1, Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1))).addMapping("type", "test", "type=text,fielddata=true"));
    Client client = client();
    // index some different terms so we have some field data for loading
    int docCount = scaledRandomIntBetween(300, 1000);
    List<IndexRequestBuilder> reqs = new ArrayList<>();
    for (long id = 0; id < docCount; id++) {
        reqs.add(client.prepareIndex("cb-test", "type", Long.toString(id)).setSource("test", "value" + id));
    }
    indexRandom(true, reqs);
    Settings resetSettings = Settings.builder().put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10b").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0).build();
    assertAcked(client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
    // Perform a search to load field data for the "test" field
    try {
        client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
        fail("should have thrown an exception");
    } catch (Exception e) {
        String errMsg = "CircuitBreakingException[[fielddata] Data too large, data for [test] would be";
        assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
        errMsg = "which is larger than the limit of [10/10b]]";
        assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
    }
    // execute a search that loads field data (sorting on the "test" field)
    // again, this time it should trip the breaker
    SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
    String errMsg = "Data too large, data for [test] would be";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    errMsg = "which is larger than the limit of [10/10b]";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    reset();
    // Adjust settings so the parent breaker will fail, but neither the fielddata breaker nor the node request breaker will fail
    resetSettings = Settings.builder().put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "500b").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "90%").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0).build();
    client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings).execute().actionGet();
    // Perform a search to load field data for the "test" field
    try {
        SearchResponse searchResponse = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
        if (searchResponse.getShardFailures().length > 0) {
            // each shard must have failed with CircuitBreakingException
            for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) {
                Throwable cause = ExceptionsHelper.unwrap(shardSearchFailure.getCause(), CircuitBreakingException.class);
                assertThat(cause, instanceOf(CircuitBreakingException.class));
                assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
            }
        } else {
            fail("should have thrown a CircuitBreakingException");
        }
    } catch (Exception e) {
        Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
        assertThat(cause, instanceOf(CircuitBreakingException.class));
        assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
        assertThat("Exception: [" + cause.toString() + "] should be caused by the parent circuit breaker", cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large"));
    }
    reset();
}

Example 2 with CircuitBreakingException

use of org.elasticsearch.common.breaker.CircuitBreakingException in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testCustomCircuitBreakerRegistration.

public void testCustomCircuitBreakerRegistration() throws Exception {
    Iterable<CircuitBreakerService> serviceIter = internalCluster().getInstances(CircuitBreakerService.class);
    final String breakerName = "customBreaker";
    BreakerSettings breakerSettings = new BreakerSettings(breakerName, 8, 1.03);
    CircuitBreaker breaker = null;
    for (CircuitBreakerService s : serviceIter) {
        s.registerBreaker(breakerSettings);
        breaker = s.getBreaker(breakerSettings.getName());
    }
    if (breaker != null) {
        try {
            breaker.addEstimateBytesAndMaybeBreak(16, "test");
        } catch (CircuitBreakingException e) {
        // ignore, we forced a circuit break
        }
    }
    NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().clear().setBreaker(true).get();
    int breaks = 0;
    for (NodeStats stat : stats.getNodes()) {
        CircuitBreakerStats breakerStats = stat.getBreaker().getStats(breakerName);
        breaks += breakerStats.getTrippedCount();
    }
    assertThat(breaks, greaterThanOrEqualTo(1));
}

Example 3 with CircuitBreakingException

use of org.elasticsearch.common.breaker.CircuitBreakingException in project elasticsearch by elastic.

the class ElasticsearchExceptionTests method testFailureToAndFromXContentWithDetails.

public void testFailureToAndFromXContentWithDetails() throws IOException {
    final XContent xContent = randomFrom(XContentType.values()).xContent();
    Exception failure;
    Throwable failureCause;
    ElasticsearchException expected;
    ElasticsearchException expectedCause;
    ElasticsearchException suppressed;
    switch(randomIntBetween(0, 6)) {
        case // Simple elasticsearch exception without cause
        0:
            failure = new NoNodeAvailableException("A");
            expected = new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]");
            expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]"));
            break;
        case // Simple elasticsearch exception with headers (other metadata of type number are not parsed)
        1:
            failure = new CircuitBreakingException("B", 5_000, 2_000);
            ((ElasticsearchException) failure).addHeader("header_name", "0", "1");
            expected = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
            expected.addHeader("header_name", "0", "1");
            suppressed = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
            suppressed.addHeader("header_name", "0", "1");
            expected.addSuppressed(suppressed);
            break;
        case // Elasticsearch exception with a cause, headers and parsable metadata
        2:
            failureCause = new NullPointerException("var is null");
            failure = new ScriptException("C", failureCause, singletonList("stack"), "test", "painless");
            ((ElasticsearchException) failure).addHeader("script_name", "my_script");
            expectedCause = new ElasticsearchException("Elasticsearch exception [type=null_pointer_exception, reason=var is null]");
            expected = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]", expectedCause);
            expected.addHeader("script_name", "my_script");
            expected.addMetadata("es.lang", "painless");
            expected.addMetadata("es.script", "test");
            expected.addMetadata("es.script_stack", "stack");
            suppressed = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]");
            suppressed.addHeader("script_name", "my_script");
            suppressed.addMetadata("es.lang", "painless");
            suppressed.addMetadata("es.script", "test");
            suppressed.addMetadata("es.script_stack", "stack");
            expected.addSuppressed(suppressed);
            break;
        case // JDK exception without cause
        3:
            failure = new IllegalStateException("D");
            expected = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
            suppressed = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
            expected.addSuppressed(suppressed);
            break;
        case // JDK exception with cause
        4:
            failureCause = new RoutingMissingException("idx", "type", "id");
            failure = new RuntimeException("E", failureCause);
            expectedCause = new ElasticsearchException("Elasticsearch exception [type=routing_missing_exception, " + "reason=routing is required for [idx]/[type]/[id]]");
            expectedCause.addMetadata("es.index", "idx");
            expectedCause.addMetadata("es.index_uuid", "_na_");
            expected = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]", expectedCause);
            suppressed = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]");
            expected.addSuppressed(suppressed);
            break;
        case // Wrapped exception with cause
        5:
            failureCause = new FileAlreadyExistsException("File exists");
            failure = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 5), "F", failureCause);
            expected = new ElasticsearchException("Elasticsearch exception [type=file_already_exists_exception, reason=File exists]");
            // strangely, the wrapped exception appears as the root cause...
            suppressed = new ElasticsearchException("Elasticsearch exception [type=broadcast_shard_operation_failed_exception, " + "reason=F]");
            expected.addSuppressed(suppressed);
            break;
        case // SearchPhaseExecutionException with cause and multiple failures
        6:
            DiscoveryNode node = new DiscoveryNode("node_g", buildNewFakeTransportAddress(), Version.CURRENT);
            failureCause = new NodeClosedException(node);
            failureCause = new NoShardAvailableActionException(new ShardId("_index_g", "_uuid_g", 6), "node_g", failureCause);
            ShardSearchFailure[] shardFailures = new ShardSearchFailure[] { new ShardSearchFailure(new ParsingException(0, 0, "Parsing g", null), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61))), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62))), new ShardSearchFailure(new SearchContextMissingException(0L), null) };
            failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
            expectedCause = new ElasticsearchException("Elasticsearch exception [type=node_closed_exception, " + "reason=node closed " + node + "]");
            expectedCause = new ElasticsearchException("Elasticsearch exception [type=no_shard_available_action_exception, " + "reason=node_g]", expectedCause);
            expectedCause.addMetadata("es.index", "_index_g");
            expectedCause.addMetadata("es.index_uuid", "_uuid_g");
            expectedCause.addMetadata("es.shard", "6");
            expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " + "reason=G]", expectedCause);
            expected.addMetadata("es.phase", "phase_g");
            expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=parsing_exception, reason=Parsing g]"));
            expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=repository_exception, " + "reason=[repository_g] Repo]"));
            expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=search_context_missing_exception, " + "reason=No search context found for id [0]]"));
            break;
        default:
            throw new UnsupportedOperationException("Failed to generate randomized failure");
    }
    Exception finalFailure = failure;
    BytesReference failureBytes = XContentHelper.toXContent((builder, params) -> {
        ElasticsearchException.generateFailureXContent(builder, params, finalFailure, true);
        return builder;
    }, xContent.type(), randomBoolean());
    try (XContentParser parser = createParser(xContent, failureBytes)) {
        failureBytes = shuffleXContent(parser, randomBoolean()).bytes();
    }
    ElasticsearchException parsedFailure;
    try (XContentParser parser = createParser(xContent, failureBytes)) {
        assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
        assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
        parsedFailure = ElasticsearchException.failureFromXContent(parser);
        assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
        assertNull(parser.nextToken());
    }
    assertDeepEquals(expected, parsedFailure);
}

Example 4 with CircuitBreakingException

use of org.elasticsearch.common.breaker.CircuitBreakingException in project crate by crate.

the class OrderedLuceneBatchIteratorFactoryTest method testOrderedLuceneBatchIteratorWithMultipleCollectorsTripsCircuitBreaker.

@Test
public void testOrderedLuceneBatchIteratorWithMultipleCollectorsTripsCircuitBreaker() throws Exception {
    RowAccounting rowAccounting = mock(RowAccountingWithEstimators.class);
    CircuitBreakingException circuitBreakingException = new CircuitBreakingException("tripped circuit breaker");
    doThrow(circuitBreakingException).when(rowAccounting).accountForAndMaybeBreak(any(Row.class));
    LuceneOrderedDocCollector collector1 = createOrderedCollector(searcher1, 1);
    LuceneOrderedDocCollector collector2 = createOrderedCollector(searcher2, 2);
    BatchIterator<Row> rowBatchIterator = OrderedLuceneBatchIteratorFactory.newInstance(Arrays.asList(collector1, collector2), OrderingByPosition.rowOrdering(new int[] { 0 }, reverseFlags, nullsFirst), rowAccounting, Runnable::run, () -> 1, true);
    consumeIteratorAndVerifyResultIsException(rowBatchIterator, circuitBreakingException);
}

Example 5 with CircuitBreakingException

use of org.elasticsearch.common.breaker.CircuitBreakingException in project crate by crate.

the class InboundPipelineTests method testPipelineHandling.

public void testPipelineHandling() throws IOException {
    final List<Tuple<MessageData, Exception>> expected = new ArrayList<>();
    final List<Tuple<MessageData, Exception>> actual = new ArrayList<>();
    final List<ReleasableBytesReference> toRelease = new ArrayList<>();
    final BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
        try {
            final Header header = m.getHeader();
            final MessageData actualData;
            final Version version = header.getVersion();
            final boolean isRequest = header.isRequest();
            final long requestId = header.getRequestId();
            final boolean isCompressed = header.isCompressed();
            if (m.isShortCircuit()) {
                actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), null);
            } else if (isRequest) {
                final TestRequest request = new TestRequest(m.openOrGetStreamInput());
                actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.value);
            } else {
                final TestResponse response = new TestResponse(m.openOrGetStreamInput());
                actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.value);
            }
            actual.add(new Tuple<>(actualData, m.getException()));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    };
    final StatsTracker statsTracker = new StatsTracker();
    final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
    final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
    final String breakThisAction = "break_this_action";
    final String actionName = "actionName";
    final Predicate<String> canTripBreaker = breakThisAction::equals;
    final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
    circuitBreaker.startBreaking();
    final InboundAggregator aggregator = new InboundAggregator(() -> circuitBreaker, canTripBreaker);
    final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
    final FakeTcpChannel channel = new FakeTcpChannel();
    final int iterations = randomIntBetween(100, 500);
    long totalMessages = 0;
    long bytesReceived = 0;
    for (int i = 0; i < iterations; ++i) {
        actual.clear();
        expected.clear();
        toRelease.clear();
        try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
            while (streamOutput.size() < BYTE_THRESHOLD) {
                final Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
                final String value = randomAlphaOfLength(randomIntBetween(10, 200));
                final boolean isRequest = randomBoolean();
                final boolean isCompressed = randomBoolean();
                final long requestId = totalMessages++;
                final MessageData messageData;
                Exception expectedExceptionClass = null;
                OutboundMessage message;
                if (isRequest) {
                    if (rarely()) {
                        messageData = new MessageData(version, requestId, true, isCompressed, breakThisAction, null);
                        message = new OutboundMessage.Request(new TestRequest(value), version, breakThisAction, requestId, false, isCompressed);
                        expectedExceptionClass = new CircuitBreakingException("");
                    } else {
                        messageData = new MessageData(version, requestId, true, isCompressed, actionName, value);
                        message = new OutboundMessage.Request(new TestRequest(value), version, actionName, requestId, false, isCompressed);
                    }
                } else {
                    messageData = new MessageData(version, requestId, false, isCompressed, null, value);
                    message = new OutboundMessage.Response(new TestResponse(value), version, requestId, false, isCompressed);
                }
                expected.add(new Tuple<>(messageData, expectedExceptionClass));
                final BytesReference reference = message.serialize(new BytesStreamOutput());
                Streams.copy(reference.streamInput(), streamOutput);
            }
            final BytesReference networkBytes = streamOutput.bytes();
            int currentOffset = 0;
            while (currentOffset != networkBytes.length()) {
                final int remainingBytes = networkBytes.length() - currentOffset;
                final int bytesToRead = Math.min(randomIntBetween(1, 32 * 1024), remainingBytes);
                final BytesReference slice = networkBytes.slice(currentOffset, bytesToRead);
                try (ReleasableBytesReference reference = new ReleasableBytesReference(slice, () -> {
                })) {
                    toRelease.add(reference);
                    bytesReceived += reference.length();
                    pipeline.handleBytes(channel, reference);
                    currentOffset += bytesToRead;
                }
            }
            final int messages = expected.size();
            for (int j = 0; j < messages; ++j) {
                final Tuple<MessageData, Exception> expectedTuple = expected.get(j);
                final Tuple<MessageData, Exception> actualTuple = actual.get(j);
                final MessageData expectedMessageData = expectedTuple.v1();
                final MessageData actualMessageData = actualTuple.v1();
                assertEquals(expectedMessageData.requestId, actualMessageData.requestId);
                assertEquals(expectedMessageData.isRequest, actualMessageData.isRequest);
                assertEquals(expectedMessageData.isCompressed, actualMessageData.isCompressed);
                assertEquals(expectedMessageData.actionName, actualMessageData.actionName);
                assertEquals(expectedMessageData.value, actualMessageData.value);
                if (expectedTuple.v2() != null) {
                    assertNotNull(actualTuple.v2());
                    assertThat(actualTuple.v2(), instanceOf(expectedTuple.v2().getClass()));
                }
            }
            for (ReleasableBytesReference released : toRelease) {
                assertEquals(0, released.refCount());
            }
        }
        assertEquals(bytesReceived, statsTracker.getBytesRead());
        assertEquals(totalMessages, statsTracker.getMessagesReceived());
    }
}