Examples with IndexRequestBuilder org.elasticsearch.action.index.IndexRequestBuilder used on opensource projects

Example 46 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class IndexStatsIT method testQueryCache.

public void testQueryCache() throws Exception {
    assertAcked(client().admin().indices().prepareCreate("idx").setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).get());
    ensureGreen();
    // index docs until we have at least one doc on each shard, otherwise, our tests will not work
    // since refresh will not refresh anything on a shard that has 0 docs and its search response get cached
    int pageDocs = randomIntBetween(2, 100);
    int numDocs = 0;
    int counter = 0;
    while (true) {
        IndexRequestBuilder[] builders = new IndexRequestBuilder[pageDocs];
        for (int i = 0; i < pageDocs; ++i) {
            builders[i] = client().prepareIndex("idx", "type", Integer.toString(counter++)).setSource(jsonBuilder().startObject().field("common", "field").field("str_value", "s" + i).endObject());
        }
        indexRandom(true, builders);
        numDocs += pageDocs;
        boolean allHaveDocs = true;
        for (ShardStats stats : client().admin().indices().prepareStats("idx").setDocs(true).get().getShards()) {
            if (stats.getStats().getDocs().getCount() == 0) {
                allHaveDocs = false;
                break;
            }
        }
        if (allHaveDocs) {
            break;
        }
    }
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L));
    for (int i = 0; i < 10; i++) {
        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
    }
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), greaterThan(0L));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), greaterThan(0L));
    // index the data again...
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs; ++i) {
        builders[i] = client().prepareIndex("idx", "type", Integer.toString(i)).setSource(jsonBuilder().startObject().field("common", "field").field("str_value", "s" + i).endObject());
    }
    indexRandom(true, builders);
    refresh();
    assertBusy(new Runnable() {

        @Override
        public void run() {
            assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
        }
    });
    for (int i = 0; i < 10; i++) {
        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
    }
    // clean the cache
    client().admin().indices().prepareClearCache().setRequestCache(true).get();
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
    // test explicit request parameter
    assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
    assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
    // set the index level setting to false, and see that the reverse works
    // clean the cache
    client().admin().indices().prepareClearCache().setRequestCache(true).get();
    assertAcked(client().admin().indices().prepareUpdateSettings("idx").setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), false)));
    assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
    assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
    assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
}

Example 47 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class ESIntegTestCase method indexRandom.

/**
     * Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
     * indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
     * ids or index segment creations. Some features might have bug when a given document is the first or the last in a
     * segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index
     * layout.
     *
     * @param forceRefresh   if <tt>true</tt> all involved indices are refreshed once the documents are indexed.
     * @param dummyDocuments if <tt>true</tt> some empty dummy documents may be randomly inserted into the document list and deleted once
     *                       all documents are indexed. This is useful to produce deleted documents on the server side.
     * @param maybeFlush     if <tt>true</tt> this method may randomly execute full flushes after index operations.
     * @param builders       the documents to index.
     */
public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
    Random random = random();
    Set<String> indicesSet = new HashSet<>();
    for (IndexRequestBuilder builder : builders) {
        indicesSet.add(builder.request().index());
    }
    Set<Tuple<String, String>> bogusIds = new HashSet<>();
    if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
        builders = new ArrayList<>(builders);
        final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
        // inject some bogus docs
        final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
        final int unicodeLen = between(1, 10);
        for (int i = 0; i < numBogusDocs; i++) {
            String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
            String index = RandomPicks.randomFrom(random, indices);
            bogusIds.add(new Tuple<>(index, id));
            builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
        }
    }
    final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
    Collections.shuffle(builders, random());
    final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
    List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
    // If you are indexing just a few documents then frequently do it one at a time.  If many then frequently in bulk.
    if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
        if (frequently()) {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute(new PayloadLatchedActionListener<IndexResponse, IndexRequestBuilder>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors));
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        } else {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), false, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute().actionGet();
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        }
    } else {
        List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE, Math.max(1, (int) (builders.size() * randomDouble()))));
        logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
        for (List<IndexRequestBuilder> segmented : partition) {
            BulkRequestBuilder bulkBuilder = client().prepareBulk();
            for (IndexRequestBuilder indexRequestBuilder : segmented) {
                bulkBuilder.add(indexRequestBuilder);
            }
            BulkResponse actionGet = bulkBuilder.execute().actionGet();
            assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
        }
    }
    for (CountDownLatch operation : inFlightAsyncOperations) {
        operation.await();
    }
    final List<Exception> actualErrors = new ArrayList<>();
    for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
        if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
            // re-index if rejected
            tuple.v1().execute().actionGet();
        } else {
            actualErrors.add(tuple.v2());
        }
    }
    assertThat(actualErrors, emptyIterable());
    if (!bogusIds.isEmpty()) {
        // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
        for (Tuple<String, String> doc : bogusIds) {
            assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", DocWriteResponse.Result.DELETED, client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
        }
    }
    if (forceRefresh) {
        assertNoFailures(client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
    }
}

Example 48 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project sonarqube by SonarSource.

the class ProxyIndexRequestBuilderTest method fail_if_bad_query.

@Test
public void fail_if_bad_query() {
    IndexRequestBuilder requestBuilder = esTester.client().prepareIndex(new IndexType("unknownIndex", "unknownType"));
    try {
        requestBuilder.get();
        fail();
    } catch (Exception e) {
        assertThat(e).isInstanceOf(IllegalStateException.class);
        assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'");
    }
}

Example 49 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class CombiIT method testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType.

/**
     * Making sure that if there are multiple aggregations, working on the same field, yet require different
     * value source type, they can all still work. It used to fail as we used to cache the ValueSource by the
     * field name. If the cached value source was of type "bytes" and another aggregation on the field required to see
     * it as "numeric", it didn't work. Now we cache the Value Sources by a custom key (field name + ValueSource type)
     * so there's no conflict there.
     */
public void testMultipleAggsOnSameField_WithDifferentRequiredValueSourceType() throws Exception {
    createIndex("idx");
    IndexRequestBuilder[] builders = new IndexRequestBuilder[randomInt(30)];
    IntIntMap values = new IntIntHashMap();
    long missingValues = 0;
    for (int i = 0; i < builders.length; i++) {
        String name = "name_" + randomIntBetween(1, 10);
        if (rarely()) {
            missingValues++;
            builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder().startObject().field("name", name).endObject());
        } else {
            int value = randomIntBetween(1, 10);
            values.put(value, values.getOrDefault(value, 0) + 1);
            builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder().startObject().field("name", name).field("value", value).endObject());
        }
    }
    indexRandom(true, builders);
    ensureSearchable();
    SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
    SearchResponse response = client().prepareSearch("idx").addAggregation(missing("missing_values").field("value")).addAggregation(terms("values").field("value").collectMode(aggCollectionMode)).execute().actionGet();
    assertSearchResponse(response);
    Aggregations aggs = response.getAggregations();
    Missing missing = aggs.get("missing_values");
    assertNotNull(missing);
    assertThat(missing.getDocCount(), equalTo(missingValues));
    Terms terms = aggs.get("values");
    assertNotNull(terms);
    Collection<Terms.Bucket> buckets = terms.getBuckets();
    assertThat(buckets.size(), equalTo(values.size()));
    for (Terms.Bucket bucket : buckets) {
        values.remove(((Number) bucket.getKey()).intValue());
    }
    assertTrue(values.isEmpty());
}

Example 50 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class StressSearchServiceReaperIT method testStressReaper.

// see issue #5165 - this test fails each time without the fix in pull #5170
public void testStressReaper() throws ExecutionException, InterruptedException {
    int num = randomIntBetween(100, 150);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[num];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test", "type", "" + i).setSource("f", English.intToEnglish(i));
    }
    createIndex("test");
    indexRandom(true, builders);
    final int iterations = scaledRandomIntBetween(500, 1000);
    for (int i = 0; i < iterations; i++) {
        SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).setSize(num).get();
        assertNoFailures(searchResponse);
        assertHitCount(searchResponse, num);
    }
}