Examples with IndicesStatsResponse org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse used on opensource projects

Example 21 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class InternalEngineMergeIT method testMergesHappening.

@TestLogging("_root:DEBUG")
public void testMergesHappening() throws InterruptedException, IOException, ExecutionException {
    final int numOfShards = randomIntBetween(1, 5);
    // some settings to keep num segments low
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numOfShards).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build()));
    long id = 0;
    final int rounds = scaledRandomIntBetween(50, 300);
    logger.info("Starting rounds [{}] ", rounds);
    for (int i = 0; i < rounds; ++i) {
        final int numDocs = scaledRandomIntBetween(100, 1000);
        BulkRequestBuilder request = client().prepareBulk();
        for (int j = 0; j < numDocs; ++j) {
            request.add(Requests.indexRequest("test").type("type1").id(Long.toString(id++)).source(jsonBuilder().startObject().field("l", randomLong()).endObject()));
        }
        BulkResponse response = request.execute().actionGet();
        refresh();
        assertNoFailures(response);
        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").setSegments(true).setMerge(true).get();
        logger.info("index round [{}] - segments {}, total merges {}, current merge {}", i, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    }
    final long upperNumberSegments = 2 * numOfShards * 10;
    awaitBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
        logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
        long current = stats.getPrimaries().getMerge().getCurrent();
        long count = stats.getPrimaries().getSegments().getCount();
        return count < upperNumberSegments && current == 0;
    });
    IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
    logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    long count = stats.getPrimaries().getSegments().getCount();
    assertThat(count, Matchers.lessThanOrEqualTo(upperNumberSegments));
}

Example 22 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class RecoveryWhileUnderLoadIT method iterateAssertCount.

private void iterateAssertCount(final int numberOfShards, final int iterations, final Set<String> ids) throws Exception {
    final long numberOfDocs = ids.size();
    SearchResponse[] iterationResults = new SearchResponse[iterations];
    boolean error = false;
    for (int i = 0; i < iterations; i++) {
        SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get();
        logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse);
        iterationResults[i] = searchResponse;
        if (searchResponse.getHits().getTotalHits() != numberOfDocs) {
            error = true;
        }
    }
    if (error) {
        //Printing out shards and their doc count
        IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().get();
        for (ShardStats shardStats : indicesStatsResponse.getShards()) {
            DocsStats docsStats = shardStats.getStats().docs;
            logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
        }
        ClusterService clusterService = clusterService();
        final ClusterState state = clusterService.state();
        for (int shard = 0; shard < numberOfShards; shard++) {
            for (String id : ids) {
                ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null);
                if (docShard.id() == shard) {
                    for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
                        GetResponse response = client().prepareGet("test", "type", id).setPreference("_only_nodes:" + shardRouting.currentNodeId()).get();
                        if (response.isExists()) {
                            logger.info("missing id [{}] on shard {}", id, shardRouting);
                        }
                    }
                }
            }
        }
        //if there was an error we try to wait and see if at some point it'll get fixed
        logger.info("--> trying to wait");
        assertTrue(awaitBusy(() -> {
            boolean errorOccurred = false;
            for (int i = 0; i < iterations; i++) {
                SearchResponse searchResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get();
                if (searchResponse.getHits().getTotalHits() != numberOfDocs) {
                    errorOccurred = true;
                }
            }
            return !errorOccurred;
        }, 5, TimeUnit.MINUTES));
        assertEquals(numberOfDocs, ids.size());
    }
    //lets now make the test fail if it was supposed to fail
    for (int i = 0; i < iterations; i++) {
        assertHitCount(iterationResults[i], numberOfDocs);
    }
}

Example 23 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class RelocationIT method beforeIndexDeletion.

@Override
protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
        for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
                Optional<ShardStats> maybePrimary = Stream.of(indexShardStats.getShards()).filter(s -> s.getShardRouting().active() && s.getShardRouting().primary()).findFirst();
                if (maybePrimary.isPresent() == false) {
                    continue;
                }
                ShardStats primary = maybePrimary.get();
                final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
                assertThat(primary.getShardRouting() + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
                for (ShardStats shardStats : indexShardStats) {
                    final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                    assertThat(shardStats.getShardRouting() + " local checkpoint mismatch", seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " global checkpoint mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " max seq no mismatch", seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
                }
            }
        }
    });
}

Example 24 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class SimpleNestedIT method assertDocumentCount.

private void assertDocumentCount(String index, long numdocs) {
    IndicesStatsResponse stats = admin().indices().prepareStats(index).clear().setDocs(true).get();
    assertNoFailures(stats);
    assertThat(stats.getIndex(index).getPrimaries().docs.getCount(), is(numdocs));
}

Example 25 with IndicesStatsResponse

use of org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse in project elasticsearch by elastic.

the class SearchStatsIT method testSimpleStats.

public void testSimpleStats() throws Exception {
    // clear all stats first
    client().admin().indices().prepareStats().clear().execute().actionGet();
    final int numNodes = cluster().numDataNodes();
    assertThat(numNodes, greaterThanOrEqualTo(2));
    // we make sure each node gets at least a single shard...
    final int shardsIdx1 = randomIntBetween(1, 10);
    final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10));
    assertThat(numNodes, lessThanOrEqualTo(shardsIdx1 + shardsIdx2));
    assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx1).put(SETTING_NUMBER_OF_REPLICAS, 0)));
    int docsTest1 = scaledRandomIntBetween(3 * shardsIdx1, 5 * shardsIdx1);
    for (int i = 0; i < docsTest1; i++) {
        client().prepareIndex("test1", "type", Integer.toString(i)).setSource("field", "value").execute().actionGet();
        if (rarely()) {
            refresh();
        }
    }
    assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx2).put(SETTING_NUMBER_OF_REPLICAS, 0)));
    int docsTest2 = scaledRandomIntBetween(3 * shardsIdx2, 5 * shardsIdx2);
    for (int i = 0; i < docsTest2; i++) {
        client().prepareIndex("test2", "type", Integer.toString(i)).setSource("field", "value").execute().actionGet();
        if (rarely()) {
            refresh();
        }
    }
    assertThat(shardsIdx1 + shardsIdx2, equalTo(numAssignedShards("test1", "test2")));
    assertThat(numAssignedShards("test1", "test2"), greaterThanOrEqualTo(2));
    // THERE WILL BE AT LEAST 2 NODES HERE SO WE CAN WAIT FOR GREEN
    ensureGreen();
    refresh();
    int iters = scaledRandomIntBetween(100, 150);
    for (int i = 0; i < iters; i++) {
        SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().prepareSearch().setQuery(QueryBuilders.termQuery("field", "value")).setStats("group1", "group2").highlighter(new HighlightBuilder().field("field")).addScriptField("script1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_source.field", Collections.emptyMap())).setSize(100).execute().actionGet();
        assertHitCount(searchResponse, docsTest1 + docsTest2);
        assertAllSuccessful(searchResponse);
    }
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    logger.debug("###### indices search stats: {}", indicesStats.getTotal().getSearch());
    assertThat(indicesStats.getTotal().getSearch().getTotal().getQueryCount(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getTotal().getQueryTimeInMillis(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getTotal().getFetchCount(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getTotal().getFetchTimeInMillis(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getGroupStats(), nullValue());
    indicesStats = client().admin().indices().prepareStats().setGroups("group1").execute().actionGet();
    assertThat(indicesStats.getTotal().getSearch().getGroupStats(), notNullValue());
    assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getQueryCount(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getQueryTimeInMillis(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getFetchCount(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getFetchTimeInMillis(), greaterThan(0L));
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
    Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
    int num = 0;
    for (NodeStats stat : nodeStats.getNodes()) {
        Stats total = stat.getIndices().getSearch().getTotal();
        if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
            assertThat(total.getQueryCount(), greaterThan(0L));
            assertThat(total.getQueryTimeInMillis(), greaterThan(0L));
            num++;
        } else {
            assertThat(total.getQueryCount(), equalTo(0L));
            assertThat(total.getQueryTimeInMillis(), equalTo(0L));
        }
    }
    assertThat(num, greaterThan(0));
}