Search in sources :

Example 31 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class RecoveryWhileUnderLoadIT method iterateAssertCount.

private void iterateAssertCount(final int numberOfShards, final int iterations, final Set<String> ids) throws Exception {
    final long numberOfDocs = ids.size();
    SearchResponse[] iterationResults = new SearchResponse[iterations];
    boolean error = false;
    for (int i = 0; i < iterations; i++) {
        SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).setTrackTotalHits(true).addSort("id", SortOrder.ASC).get();
        logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse);
        iterationResults[i] = searchResponse;
        if (searchResponse.getHits().getTotalHits().value != numberOfDocs) {
            error = true;
        }
    }
    if (error) {
        // Printing out shards and their doc count
        IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().get();
        for (ShardStats shardStats : indicesStatsResponse.getShards()) {
            DocsStats docsStats = shardStats.getStats().docs;
            logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
        }
        ClusterService clusterService = clusterService();
        final ClusterState state = clusterService.state();
        for (int shard = 0; shard < numberOfShards; shard++) {
            for (String id : ids) {
                ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null);
                if (docShard.id() == shard) {
                    for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
                        GetResponse response = client().prepareGet("test", id).setPreference("_only_nodes:" + shardRouting.currentNodeId()).get();
                        if (response.isExists()) {
                            logger.info("missing id [{}] on shard {}", id, shardRouting);
                        }
                    }
                }
            }
        }
        // if there was an error we try to wait and see if at some point it'll get fixed
        logger.info("--> trying to wait");
        assertBusy(() -> {
            boolean errorOccurred = false;
            for (int i = 0; i < iterations; i++) {
                SearchResponse searchResponse = client().prepareSearch().setTrackTotalHits(true).setSize(0).setQuery(matchAllQuery()).get();
                if (searchResponse.getHits().getTotalHits().value != numberOfDocs) {
                    errorOccurred = true;
                }
            }
            assertFalse("An error occurred while waiting", errorOccurred);
        }, 5, TimeUnit.MINUTES);
        assertEquals(numberOfDocs, ids.size());
    }
    // lets now make the test fail if it was supposed to fail
    for (int i = 0; i < iterations; i++) {
        assertHitCount(iterationResults[i], numberOfDocs);
    }
}
Also used : ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) IndicesStatsResponse(org.opensearch.action.admin.indices.stats.IndicesStatsResponse) ClusterState(org.opensearch.cluster.ClusterState) GetResponse(org.opensearch.action.get.GetResponse) SearchResponse(org.opensearch.action.search.SearchResponse) ShardId(org.opensearch.index.shard.ShardId) ClusterService(org.opensearch.cluster.service.ClusterService) DocsStats(org.opensearch.index.shard.DocsStats) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 32 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class TransportRolloverActionTests method randomIndicesStatsResponse.

public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadata[] indices) {
    List<ShardStats> shardStats = new ArrayList<>();
    for (final IndexMetadata index : indices) {
        int numShards = randomIntBetween(1, 3);
        // -1 means there is no primary shard.
        int primaryIdx = randomIntBetween(-1, numShards - 1);
        for (int i = 0; i < numShards; i++) {
            ShardId shardId = new ShardId(index.getIndex(), i);
            boolean primary = (i == primaryIdx);
            Path path = createTempDir().resolve("indices").resolve(index.getIndexUUID()).resolve(String.valueOf(i));
            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary, primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
            shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
            shardRouting = shardRouting.moveToStarted();
            CommonStats stats = new CommonStats();
            stats.fieldData = new FieldDataStats();
            stats.queryCache = new QueryCacheStats();
            stats.docs = new DocsStats();
            stats.store = new StoreStats();
            stats.indexing = new IndexingStats();
            stats.search = new SearchStats();
            stats.segments = new SegmentsStats();
            stats.merge = new MergeStats();
            stats.refresh = new RefreshStats();
            stats.completion = new CompletionStats();
            stats.requestCache = new RequestCacheStats();
            stats.get = new GetStats();
            stats.flush = new FlushStats();
            stats.warmer = new WarmerStats();
            shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null));
        }
    }
    return IndicesStatsTests.newIndicesStatsResponse(shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList());
}
Also used : StoreStats(org.opensearch.index.store.StoreStats) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) RefreshStats(org.opensearch.index.refresh.RefreshStats) ArrayList(java.util.ArrayList) GetStats(org.opensearch.index.get.GetStats) SegmentsStats(org.opensearch.index.engine.SegmentsStats) ShardId(org.opensearch.index.shard.ShardId) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) FlushStats(org.opensearch.index.flush.FlushStats) ShardPath(org.opensearch.index.shard.ShardPath) QueryCacheStats(org.opensearch.index.cache.query.QueryCacheStats) DocsStats(org.opensearch.index.shard.DocsStats) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) Path(java.nio.file.Path) ShardPath(org.opensearch.index.shard.ShardPath) IndexingStats(org.opensearch.index.shard.IndexingStats) WarmerStats(org.opensearch.index.warmer.WarmerStats) SearchStats(org.opensearch.index.search.stats.SearchStats) MergeStats(org.opensearch.index.merge.MergeStats) ShardRouting(org.opensearch.cluster.routing.ShardRouting) RequestCacheStats(org.opensearch.index.cache.request.RequestCacheStats) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats)

Aggregations

ShardStats (org.opensearch.action.admin.indices.stats.ShardStats)32 IndicesStatsResponse (org.opensearch.action.admin.indices.stats.IndicesStatsResponse)16 ShardRouting (org.opensearch.cluster.routing.ShardRouting)15 Settings (org.opensearch.common.settings.Settings)15 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)13 ShardId (org.opensearch.index.shard.ShardId)13 Arrays (java.util.Arrays)12 OpenSearchIntegTestCase (org.opensearch.test.OpenSearchIntegTestCase)12 OpenSearchAssertions.assertAcked (org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked)12 ArrayList (java.util.ArrayList)11 CountDownLatch (java.util.concurrent.CountDownLatch)11 ClusterState (org.opensearch.cluster.ClusterState)11 Collection (java.util.Collection)10 List (java.util.List)10 Matchers.equalTo (org.hamcrest.Matchers.equalTo)10 TimeValue (org.opensearch.common.unit.TimeValue)10 IndexShard (org.opensearch.index.shard.IndexShard)10 IndicesService (org.opensearch.indices.IndicesService)10 Plugin (org.opensearch.plugins.Plugin)10 MockTransportService (org.opensearch.test.transport.MockTransportService)10