Examples with ShardSearchFailure org.elasticsearch.action.search.ShardSearchFailure used on opensource projects

Example 1 with ShardSearchFailure

use of org.elasticsearch.action.search.ShardSearchFailure in project elasticsearch by elastic.

the class RejectionActionIT method testSimulatedSearchRejectionLoad.

public void testSimulatedSearchRejectionLoad() throws Throwable {
    for (int i = 0; i < 10; i++) {
        client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "1").get();
    }
    int numberOfAsyncOps = randomIntBetween(200, 700);
    final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps);
    final CopyOnWriteArrayList<Object> responses = new CopyOnWriteArrayList<>();
    for (int i = 0; i < numberOfAsyncOps; i++) {
        client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(QueryBuilders.matchQuery("field", "1")).execute(new ActionListener<SearchResponse>() {

            @Override
            public void onResponse(SearchResponse searchResponse) {
                responses.add(searchResponse);
                latch.countDown();
            }

            @Override
            public void onFailure(Exception e) {
                responses.add(e);
                latch.countDown();
            }
        });
    }
    latch.await();
    // validate all responses
    for (Object response : responses) {
        if (response instanceof SearchResponse) {
            SearchResponse searchResponse = (SearchResponse) response;
            for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
                assertTrue("got unexpected reason..." + failure.reason(), failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
            }
        } else {
            Exception t = (Exception) response;
            Throwable unwrap = ExceptionsHelper.unwrapCause(t);
            if (unwrap instanceof SearchPhaseExecutionException) {
                SearchPhaseExecutionException e = (SearchPhaseExecutionException) unwrap;
                for (ShardSearchFailure failure : e.shardFailures()) {
                    assertTrue("got unexpected reason..." + failure.reason(), failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
                }
            } else if ((unwrap instanceof EsRejectedExecutionException) == false) {
                throw new AssertionError("unexpected failure", (Throwable) response);
            }
        }
    }
    assertThat(responses.size(), equalTo(numberOfAsyncOps));
}

Example 2 with ShardSearchFailure

use of org.elasticsearch.action.search.ShardSearchFailure in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testParentChecking.

/**
     * Test that a breaker correctly redistributes to a different breaker, in
     * this case, the fielddata breaker borrows space from the request breaker
     */
@TestLogging("_root:DEBUG,org.elasticsearch.action.search:TRACE")
public void testParentChecking() throws Exception {
    if (noopBreakerUsed()) {
        logger.info("--> noop breakers used, skipping test");
        return;
    }
    assertAcked(prepareCreate("cb-test", 1, Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1))).addMapping("type", "test", "type=text,fielddata=true"));
    Client client = client();
    // index some different terms so we have some field data for loading
    int docCount = scaledRandomIntBetween(300, 1000);
    List<IndexRequestBuilder> reqs = new ArrayList<>();
    for (long id = 0; id < docCount; id++) {
        reqs.add(client.prepareIndex("cb-test", "type", Long.toString(id)).setSource("test", "value" + id));
    }
    indexRandom(true, reqs);
    Settings resetSettings = Settings.builder().put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10b").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0).build();
    assertAcked(client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
    // Perform a search to load field data for the "test" field
    try {
        client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
        fail("should have thrown an exception");
    } catch (Exception e) {
        String errMsg = "CircuitBreakingException[[fielddata] Data too large, data for [test] would be";
        assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
        errMsg = "which is larger than the limit of [10/10b]]";
        assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
    }
    // execute a search that loads field data (sorting on the "test" field)
    // again, this time it should trip the breaker
    SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
    String errMsg = "Data too large, data for [test] would be";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    errMsg = "which is larger than the limit of [10/10b]";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    reset();
    // Adjust settings so the parent breaker will fail, but neither the fielddata breaker nor the node request breaker will fail
    resetSettings = Settings.builder().put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "500b").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "90%").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0).build();
    client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings).execute().actionGet();
    // Perform a search to load field data for the "test" field
    try {
        SearchResponse searchResponse = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
        if (searchResponse.getShardFailures().length > 0) {
            // each shard must have failed with CircuitBreakingException
            for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) {
                Throwable cause = ExceptionsHelper.unwrap(shardSearchFailure.getCause(), CircuitBreakingException.class);
                assertThat(cause, instanceOf(CircuitBreakingException.class));
                assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
            }
        } else {
            fail("should have thrown a CircuitBreakingException");
        }
    } catch (Exception e) {
        Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
        assertThat(cause, instanceOf(CircuitBreakingException.class));
        assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
        assertThat("Exception: [" + cause.toString() + "] should be caused by the parent circuit breaker", cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large"));
    }
    reset();
}

Example 3 with ShardSearchFailure

use of org.elasticsearch.action.search.ShardSearchFailure in project elasticsearch by elastic.

the class IndexLookupIT method checkExceptions.

private void checkExceptions(Script script) {
    try {
        SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script).execute().actionGet();
        assertThat(sr.getHits().getHits().length, equalTo(0));
        ShardSearchFailure[] shardFails = sr.getShardFailures();
        for (ShardSearchFailure fail : shardFails) {
            assertThat(fail.reason().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), Matchers.greaterThan(-1));
        }
    } catch (SearchPhaseExecutionException ex) {
        assertThat("got " + ex.toString(), ex.toString().indexOf("Cannot iterate twice! If you want to iterate more that once, add _CACHE explicitly."), Matchers.greaterThan(-1));
    }
}

Example 4 with ShardSearchFailure

use of org.elasticsearch.action.search.ShardSearchFailure in project elasticsearch by elastic.

the class IndexLookupIT method testCallWithDifferentFlagsFails.

public void testCallWithDifferentFlagsFails() throws Exception {
    initTestData();
    final int numPrimaries = getNumShards("test").numPrimaries;
    final String expectedError = "You must call get with all required flags! " + "Instead of  _index['int_payload_field'].get('b', _FREQUENCIES) and _index['int_payload_field'].get('b', _POSITIONS)" + " call  _index['int_payload_field'].get('b', _FREQUENCIES | _POSITIONS)  once]";
    // should throw an exception, we cannot call with different flags twice
    // if the flags of the second call were not included in the first call.
    Script script = createScript("Call with different flags twice");
    try {
        SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script).get();
        // (partial) success when at least one shard succeeds
        assertThat(numPrimaries, greaterThan(response.getShardFailures().length));
        assertThat(response.getFailedShards(), greaterThanOrEqualTo(1));
        for (ShardSearchFailure failure : response.getShardFailures()) {
            assertThat(failure.reason(), containsString(expectedError));
        }
    } catch (SearchPhaseExecutionException e) {
        // Exception thrown when *all* shards fail
        assertThat(numPrimaries, equalTo(e.shardFailures().length));
        for (ShardSearchFailure failure : e.shardFailures()) {
            assertThat(failure.reason(), containsString(expectedError));
        }
    }
    // Should not throw an exception this way round
    script = createScript("Call with same flags twice");
    assertThat(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).addScriptField("tvtest", script).get().getHits().getTotalHits(), greaterThan(0L));
}

Example 5 with ShardSearchFailure

use of org.elasticsearch.action.search.ShardSearchFailure in project elasticsearch by elastic.

the class SnapshotsService method endSnapshot.

/**
     * Finalizes the shard in repository and then removes it from cluster state
     * <p>
     * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
     *
     * @param entry   snapshot
     * @param failure failure reason or null if snapshot was successful
     */
private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {

        @Override
        public void run() {
            final Snapshot snapshot = entry.snapshot();
            try {
                final Repository repository = repositoriesService.repository(snapshot.getRepository());
                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
                ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
                    ShardId shardId = shardStatus.key;
                    ShardSnapshotStatus status = shardStatus.value;
                    if (status.state().failed()) {
                        failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId)));
                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                    }
                }
                SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures), entry.getRepositoryStateId());
                removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
            } catch (Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
                removeSnapshotFromClusterState(snapshot, null, e);
            }
        }
    });
}