Examples with IndexResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse used on opensource projects

Example 76 with IndexResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse in project elasticsearch by elastic.

the class IndexStatsIT method testConcurrentIndexingAndStatsRequests.

/**
     * Test that we can safely concurrently index and get stats. This test was inspired by a serialization issue that arose due to a race
     * getting doc stats during heavy indexing. The race could lead to deleted docs being negative which would then be serialized as a
     * variable-length long. Since serialization of negative longs using a variable-length format was unsupported
     * ({@link org.elasticsearch.common.io.stream.StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test
     * that we can continue to get stats while indexing.
     */
public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierException, InterruptedException, ExecutionException {
    final AtomicInteger idGenerator = new AtomicInteger();
    final int numberOfIndexingThreads = Runtime.getRuntime().availableProcessors();
    final int numberOfStatsThreads = 4 * numberOfIndexingThreads;
    final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfIndexingThreads + numberOfStatsThreads);
    final AtomicBoolean stop = new AtomicBoolean();
    final List<Thread> threads = new ArrayList<>(numberOfIndexingThreads + numberOfIndexingThreads);
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicBoolean failed = new AtomicBoolean();
    final AtomicReference<List<ShardOperationFailedException>> shardFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
    final AtomicReference<List<Exception>> executionFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
    // increasing the number of shards increases the number of chances any one stats request will hit a race
    final CreateIndexRequest createIndexRequest = new CreateIndexRequest("test", Settings.builder().put("index.number_of_shards", 10).build());
    client().admin().indices().create(createIndexRequest).get();
    // start threads that will index concurrently with stats requests
    for (int i = 0; i < numberOfIndexingThreads; i++) {
        final Thread thread = new Thread(() -> {
            try {
                barrier.await();
            } catch (final BrokenBarrierException | InterruptedException e) {
                failed.set(true);
                executionFailures.get().add(e);
                latch.countDown();
            }
            while (!stop.get()) {
                final String id = Integer.toString(idGenerator.incrementAndGet());
                final IndexResponse response = client().prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).get();
                assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED));
            }
        });
        thread.setName("indexing-" + i);
        threads.add(thread);
        thread.start();
    }
    // start threads that will get stats concurrently with indexing
    for (int i = 0; i < numberOfStatsThreads; i++) {
        final Thread thread = new Thread(() -> {
            try {
                barrier.await();
            } catch (final BrokenBarrierException | InterruptedException e) {
                failed.set(true);
                executionFailures.get().add(e);
                latch.countDown();
            }
            final IndicesStatsRequest request = new IndicesStatsRequest();
            request.all();
            request.indices(new String[0]);
            while (!stop.get()) {
                try {
                    final IndicesStatsResponse response = client().admin().indices().stats(request).get();
                    if (response.getFailedShards() > 0) {
                        failed.set(true);
                        shardFailures.get().addAll(Arrays.asList(response.getShardFailures()));
                        latch.countDown();
                    }
                } catch (final ExecutionException | InterruptedException e) {
                    failed.set(true);
                    executionFailures.get().add(e);
                    latch.countDown();
                }
            }
        });
        thread.setName("stats-" + i);
        threads.add(thread);
        thread.start();
    }
    // release the hounds
    barrier.await();
    // wait for a failure, or for fifteen seconds to elapse
    latch.await(15, TimeUnit.SECONDS);
    // stop all threads and wait for them to complete
    stop.set(true);
    for (final Thread thread : threads) {
        thread.join();
    }
    assertThat(shardFailures.get(), emptyCollectionOf(ShardOperationFailedException.class));
    assertThat(executionFailures.get(), emptyCollectionOf(Exception.class));
}

Example 77 with IndexResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse in project elasticsearch by elastic.

the class ConcurrentDynamicTemplateIT method testConcurrentDynamicMapping.

// see #3544
public void testConcurrentDynamicMapping() throws Exception {
    final String fieldName = "field";
    final String mapping = "{ \"" + mappingType + "\": {" + "\"dynamic_templates\": [" + "{ \"" + fieldName + "\": {" + "\"path_match\": \"*\"," + "\"mapping\": {" + "\"type\": \"text\"," + "\"store\": true," + "\"analyzer\": \"whitespace\" } } } ] } }";
    // The 'fieldNames' array is used to help with retrieval of index terms
    // after testing
    int iters = scaledRandomIntBetween(5, 15);
    for (int i = 0; i < iters; i++) {
        cluster().wipeIndices("test");
        assertAcked(prepareCreate("test").addMapping(mappingType, mapping, XContentType.JSON));
        int numDocs = scaledRandomIntBetween(10, 100);
        final CountDownLatch latch = new CountDownLatch(numDocs);
        final List<Throwable> throwable = new CopyOnWriteArrayList<>();
        int currentID = 0;
        for (int j = 0; j < numDocs; j++) {
            Map<String, Object> source = new HashMap<>();
            source.put(fieldName, "test-user");
            client().prepareIndex("test", mappingType, Integer.toString(currentID++)).setSource(source).execute(new ActionListener<IndexResponse>() {

                @Override
                public void onResponse(IndexResponse response) {
                    latch.countDown();
                }

                @Override
                public void onFailure(Exception e) {
                    throwable.add(e);
                    latch.countDown();
                }
            });
        }
        latch.await();
        assertThat(throwable, emptyIterable());
        refresh();
        assertHitCount(client().prepareSearch("test").setQuery(QueryBuilders.matchQuery(fieldName, "test-user")).get(), numDocs);
        assertHitCount(client().prepareSearch("test").setQuery(QueryBuilders.matchQuery(fieldName, "test user")).get(), 0);
    }
}

Example 78 with IndexResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse in project elasticsearch by elastic.

the class SearchWithRandomExceptionsIT method testRandomExceptions.

public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test").field("type", "keyword").endObject().endObject().endObject().endObject().string();
    final double lowLevelRate;
    final double topLevelRate;
    if (frequently()) {
        if (randomBoolean()) {
            if (randomBoolean()) {
                lowLevelRate = 1.0 / between(2, 10);
                topLevelRate = 0.0d;
            } else {
                topLevelRate = 1.0 / between(2, 10);
                lowLevelRate = 0.0d;
            }
        } else {
            lowLevelRate = 1.0 / between(2, 10);
            topLevelRate = 1.0 / between(2, 10);
        }
    } else {
        // rarely no exception
        topLevelRate = 0d;
        lowLevelRate = 0d;
    }
    Builder settings = Settings.builder().put(indexSettings()).put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate).put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate).put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
    logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
    assertAcked(prepareCreate("test").setSettings(settings).addMapping("type", mapping, XContentType.JSON));
    ensureSearchable();
    final int numDocs = between(10, 100);
    int numCreated = 0;
    boolean[] added = new boolean[numDocs];
    for (int i = 0; i < numDocs; i++) {
        try {
            IndexResponse indexResponse = client().prepareIndex("test", "type", "" + i).setTimeout(TimeValue.timeValueSeconds(1)).setSource("test", English.intToEnglish(i)).get();
            if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
                numCreated++;
                added[i] = true;
            }
        } catch (ElasticsearchException ex) {
        }
    }
    logger.info("Start Refresh");
    // don't assert on failures here
    RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().get();
    final boolean refreshFailed = refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info("Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ", refreshFailed, refreshResponse.getFailedShards(), refreshResponse.getShardFailures().length, refreshResponse.getSuccessfulShards(), refreshResponse.getTotalShards());
    NumShards test = getNumShards("test");
    final int numSearches = scaledRandomIntBetween(100, 200);
    // we don't check anything here really just making sure we don't leave any open files or a broken index behind.
    for (int i = 0; i < numSearches; i++) {
        try {
            int docToQuery = between(0, numDocs - 1);
            int expectedResults = added[docToQuery] ? 1 : 0;
            logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
            SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery))).setSize(expectedResults).get();
            logger.info("Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
            if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
                assertResultsAndLogOnFailure(expectedResults, searchResponse);
            }
            // check match all
            searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).setSize(numCreated).addSort("_id", SortOrder.ASC).get();
            logger.info("Match all Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
            if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
                assertResultsAndLogOnFailure(numCreated, searchResponse);
            }
        } catch (SearchPhaseExecutionException ex) {
            logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
    }
}

Example 79 with IndexResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse in project elasticsearch by elastic.

the class TaskResultsService method doStoreResult.

private void doStoreResult(TaskResult taskResult, ActionListener<Void> listener) {
    IndexRequestBuilder index = client.prepareIndex(TASK_INDEX, TASK_TYPE, taskResult.getTask().getTaskId().toString());
    try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
        taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
        index.setSource(builder);
    } catch (IOException e) {
        throw new ElasticsearchException("Couldn't convert task result to XContent for [{}]", e, taskResult.getTask());
    }
    index.execute(new ActionListener<IndexResponse>() {

        @Override
        public void onResponse(IndexResponse indexResponse) {
            listener.onResponse(null);
        }

        @Override
        public void onFailure(Exception e) {
            listener.onFailure(e);
        }
    });
}

Example 80 with IndexResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.index.IndexResponse in project elasticsearch by elastic.

the class IndexAliasesIT method testAliases.

public void testAliases() throws Exception {
    logger.info("--> creating index [test]");
    createIndex("test");
    ensureGreen();
    logger.info("--> aliasing index [test] with [alias1]");
    assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1"));
    logger.info("--> indexing against [alias1], should work now");
    IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet();
    assertThat(indexResponse.getIndex(), equalTo("test"));
    logger.info("--> creating index [test_x]");
    createIndex("test_x");
    ensureGreen();
    logger.info("--> remove [alias1], Aliasing index [test_x] with [alias1]");
    assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1"));
    logger.info("--> indexing against [alias1], should work against [test_x]");
    indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet();
    assertThat(indexResponse.getIndex(), equalTo("test_x"));
}