Examples with IndicesStatsResponse org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse used on opensource projects

Example 6 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class SearchStatsIT method testOpenContexts.

public void testOpenContexts() {
    String index = "test1";
    createIndex(index);
    ensureGreen(index);
    // create shards * docs number of docs and attempt to distribute them equally
    // this distribution will not be perfect; each shard will have an integer multiple of docs (possibly zero)
    // we do this so we have a lot of pages to scroll through
    final int docs = scaledRandomIntBetween(20, 50);
    for (int s = 0; s < numAssignedShards(index); s++) {
        for (int i = 0; i < docs; i++) {
            client().prepareIndex(index, "type", Integer.toString(s * docs + i)).setSource("field", "value").setRouting(Integer.toString(s)).execute().actionGet();
        }
    }
    client().admin().indices().prepareRefresh(index).execute().actionGet();
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats(index).execute().actionGet();
    assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0L));
    int size = scaledRandomIntBetween(1, docs);
    SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).setScroll(TimeValue.timeValueMinutes(2)).execute().actionGet();
    assertSearchResponse(searchResponse);
    // refresh the stats now that scroll contexts are opened
    indicesStats = client().admin().indices().prepareStats(index).execute().actionGet();
    assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo((long) numAssignedShards(index)));
    assertThat(indicesStats.getTotal().getSearch().getTotal().getScrollCurrent(), equalTo((long) numAssignedShards(index)));
    int hits = 0;
    while (true) {
        if (searchResponse.getHits().getHits().length == 0) {
            break;
        }
        hits += searchResponse.getHits().getHits().length;
        searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).execute().actionGet();
    }
    long expected = 0;
    // the number of queries executed is equal to at least the sum of number of pages in shard over all shards
    IndicesStatsResponse r = client().admin().indices().prepareStats(index).execute().actionGet();
    for (int s = 0; s < numAssignedShards(index); s++) {
        expected += (long) Math.ceil(r.getShards()[s].getStats().getDocs().getCount() / size);
    }
    indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    Stats stats = indicesStats.getTotal().getSearch().getTotal();
    assertEquals(hits, docs * numAssignedShards(index));
    assertThat(stats.getQueryCount(), greaterThanOrEqualTo(expected));
    clearScroll(searchResponse.getScrollId());
    indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    stats = indicesStats.getTotal().getSearch().getTotal();
    assertThat(indicesStats.getTotal().getSearch().getOpenContexts(), equalTo(0L));
    assertThat(stats.getScrollCount(), equalTo((long) numAssignedShards(index)));
    assertThat(stats.getScrollTimeInMillis(), greaterThan(0L));
}

Example 7 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class IndexWithShadowReplicasIT method testIndexWithFewDocuments.

@TestLogging("org.elasticsearch.gateway:TRACE")
public void testIndexWithFewDocuments() throws Exception {
    final Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);
    internalCluster().startNodes(3, nodeSettings);
    final String IDX = "test";
    Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()).put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).build();
    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    ensureGreen(IDX);
    // So basically, the primary should fail and the replica will need to
    // replay the translog, this is what this tests
    client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
    IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(IDX).clear().setTranslog(true).get();
    assertEquals(2, indicesStatsResponse.getIndex(IDX).getPrimaries().getTranslog().estimatedNumberOfOperations());
    assertEquals(2, indicesStatsResponse.getIndex(IDX).getTotal().getTranslog().estimatedNumberOfOperations());
    Index index = resolveIndex(IDX);
    for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
        IndexService indexService = service.indexService(index);
        if (indexService != null) {
            IndexShard shard = indexService.getShard(0);
            TranslogStats translogStats = shard.translogStats();
            assertTrue(translogStats != null || shard instanceof ShadowIndexShard);
            if (translogStats != null) {
                assertEquals(2, translogStats.estimatedNumberOfOperations());
            }
        }
    }
    // Check that we can get doc 1 and 2, because we are doing realtime
    // gets and getting from the primary
    GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
    GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
    flushAndRefresh(IDX);
    client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get();
    refresh();
    // Check that we can get doc 1 and 2 without realtime
    gResp1 = client().prepareGet(IDX, "doc", "1").setRealtime(false).get();
    gResp2 = client().prepareGet(IDX, "doc", "2").setRealtime(false).get();
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
    logger.info("--> restarting all nodes");
    if (randomBoolean()) {
        logger.info("--> rolling restart");
        internalCluster().rollingRestart();
    } else {
        logger.info("--> full restart");
        internalCluster().fullRestart();
    }
    client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
    ensureGreen(IDX);
    flushAndRefresh(IDX);
    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
    assertHitCount(resp, 4);
    logger.info("--> deleting index");
    assertAcked(client().admin().indices().prepareDelete(IDX));
}

Example 8 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 9 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class SuggestStatsIT method testSimpleStats.

public void testSimpleStats() throws Exception {
    // clear all stats first
    client().admin().indices().prepareStats().clear().execute().actionGet();
    final int numNodes = cluster().numDataNodes();
    assertThat(numNodes, greaterThanOrEqualTo(2));
    // we make sure each node gets at least a single shard...
    final int shardsIdx1 = randomIntBetween(1, 10);
    final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10));
    final int totalShards = shardsIdx1 + shardsIdx2;
    assertThat(numNodes, lessThanOrEqualTo(totalShards));
    assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx1).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx2).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertThat(shardsIdx1 + shardsIdx2, equalTo(numAssignedShards("test1", "test2")));
    assertThat(numAssignedShards("test1", "test2"), greaterThanOrEqualTo(2));
    ensureGreen();
    for (int i = 0; i < randomIntBetween(20, 100); i++) {
        index("test" + ((i % 2) + 1), "type", "" + i, "f", "test" + i);
    }
    refresh();
    int suggestAllIdx = scaledRandomIntBetween(20, 50);
    int suggestIdx1 = scaledRandomIntBetween(20, 50);
    int suggestIdx2 = scaledRandomIntBetween(20, 50);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < suggestAllIdx; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch(), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx1; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test1"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx2; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test2"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    long endTime = System.currentTimeMillis();
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal();
    // check current
    assertThat(suggest.getSuggestCurrent(), equalTo(0L));
    // check suggest count
    assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2)));
    assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1)));
    assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2)));
    logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime);
    // check suggest time
    assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L));
    // the upperbound is num shards * total time since we do searches in parallel
    assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
    Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
    int num = 0;
    for (NodeStats stat : nodeStats.getNodes()) {
        SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal();
        logger.info("evaluating {}", stat.getNode());
        if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
            assertThat(suggestStats.getSuggestCount(), greaterThan(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L));
            num++;
        } else {
            assertThat(suggestStats.getSuggestCount(), equalTo(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L));
        }
    }
    assertThat(num, greaterThan(0));
}

Example 10 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class IndexStatsIT method testFilterCacheStats.

public void testFilterCacheStats() throws Exception {
    assertAcked(prepareCreate("index").setSettings(Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build()).get());
    indexRandom(true, client().prepareIndex("index", "type", "1").setSource("foo", "bar"), client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
    ensureGreen();
    IndicesStatsResponse response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
    assertCumulativeQueryCacheStats(response);
    assertEquals(0, response.getTotal().queryCache.getCacheSize());
    // the query cache has an optimization that disables it automatically if there is contention,
    // so we run it in an assertBusy block which should eventually succeed
    assertBusy(() -> {
        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
        IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(stats);
        assertThat(stats.getTotal().queryCache.getHitCount(), equalTo(0L));
        assertThat(stats.getTotal().queryCache.getEvictions(), equalTo(0L));
        assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
    });
    assertBusy(() -> {
        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
        IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(stats);
        assertThat(stats.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getEvictions(), equalTo(0L));
        assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
    });
    assertEquals(DocWriteResponse.Result.DELETED, client().prepareDelete("index", "type", "1").get().getResult());
    assertEquals(DocWriteResponse.Result.DELETED, client().prepareDelete("index", "type", "2").get().getResult());
    refresh();
    response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
    assertCumulativeQueryCacheStats(response);
    assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
    assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
    assertThat(response.getTotal().queryCache.getCacheSize(), equalTo(0L));
    assertThat(response.getTotal().queryCache.getCacheCount(), greaterThan(0L));
    indexRandom(true, client().prepareIndex("index", "type", "1").setSource("foo", "bar"), client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
    assertBusy(() -> {
        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
        IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(stats);
        assertThat(stats.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getEvictions(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
        assertThat(stats.getTotal().queryCache.getMemorySizeInBytes(), greaterThan(0L));
    });
    assertAllSuccessful(client().admin().indices().prepareClearCache("index").setQueryCache(true).get());
    response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
    assertCumulativeQueryCacheStats(response);
    assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
    assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
    assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
    assertThat(response.getTotal().queryCache.getCacheSize(), equalTo(0L));
    assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), equalTo(0L));
}