Examples with ShardStats org.elasticsearch.action.admin.indices.stats.ShardStats used on opensource projects

Example 11 with ShardStats

use of org.elasticsearch.action.admin.indices.stats.ShardStats in project graylog2-server by Graylog2.

the class Indices method getIndicesStats.

public Set<IndexStatistics> getIndicesStats(final IndexSet indexSet) {
    final Map<String, IndexStats> responseIndices;
    try {
        responseIndices = getAll(indexSet);
    } catch (ElasticsearchException e) {
        return Collections.emptySet();
    }
    final ImmutableSet.Builder<IndexStatistics> result = ImmutableSet.builder();
    for (IndexStats indexStats : responseIndices.values()) {
        final ImmutableList.Builder<ShardRouting> shardRouting = ImmutableList.builder();
        for (ShardStats shardStats : indexStats.getShards()) {
            shardRouting.add(shardStats.getShardRouting());
        }
        final IndexStatistics stats = IndexStatistics.create(indexStats.getIndex(), indexStats.getPrimaries(), indexStats.getTotal(), shardRouting.build());
        result.add(stats);
    }
    return result.build();
}

Example 12 with ShardStats

use of org.elasticsearch.action.admin.indices.stats.ShardStats in project elasticsearch by elastic.

the class IndicesService method stats.

public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
    CommonStats oldStats = new CommonStats(flags);
    if (includePrevious) {
        Flag[] setFlags = flags.getFlags();
        for (Flag flag : setFlags) {
            switch(flag) {
                case Get:
                    oldStats.get.add(oldShardsStats.getStats);
                    break;
                case Indexing:
                    oldStats.indexing.add(oldShardsStats.indexingStats);
                    break;
                case Search:
                    oldStats.search.add(oldShardsStats.searchStats);
                    break;
                case Merge:
                    oldStats.merge.add(oldShardsStats.mergeStats);
                    break;
                case Refresh:
                    oldStats.refresh.add(oldShardsStats.refreshStats);
                    break;
                case Recovery:
                    oldStats.recoveryStats.add(oldShardsStats.recoveryStats);
                    break;
                case Flush:
                    oldStats.flush.add(oldShardsStats.flushStats);
                    break;
            }
        }
    }
    Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
    for (IndexService indexService : this) {
        for (IndexShard indexShard : indexService) {
            try {
                if (indexShard.routingEntry() == null) {
                    continue;
                }
                IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats(), indexShard.seqNoStats()) });
                if (!statsByShard.containsKey(indexService.index())) {
                    statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                } else {
                    statsByShard.get(indexService.index()).add(indexShardStats);
                }
            } catch (IllegalIndexShardStateException e) {
                // we can safely ignore illegal state on ones that are closing for example
                logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} ignoring shard stats", indexShard.shardId()), e);
            }
        }
    }
    return new NodeIndicesStats(oldStats, statsByShard);
}

Example 13 with ShardStats

use of org.elasticsearch.action.admin.indices.stats.ShardStats in project elasticsearch by elastic.

the class RecoveryWhileUnderLoadIT method iterateAssertCount.

private void iterateAssertCount(final int numberOfShards, final int iterations, final Set<String> ids) throws Exception {
    final long numberOfDocs = ids.size();
    SearchResponse[] iterationResults = new SearchResponse[iterations];
    boolean error = false;
    for (int i = 0; i < iterations; i++) {
        SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get();
        logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse);
        iterationResults[i] = searchResponse;
        if (searchResponse.getHits().getTotalHits() != numberOfDocs) {
            error = true;
        }
    }
    if (error) {
        //Printing out shards and their doc count
        IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().get();
        for (ShardStats shardStats : indicesStatsResponse.getShards()) {
            DocsStats docsStats = shardStats.getStats().docs;
            logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
        }
        ClusterService clusterService = clusterService();
        final ClusterState state = clusterService.state();
        for (int shard = 0; shard < numberOfShards; shard++) {
            for (String id : ids) {
                ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null);
                if (docShard.id() == shard) {
                    for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
                        GetResponse response = client().prepareGet("test", "type", id).setPreference("_only_nodes:" + shardRouting.currentNodeId()).get();
                        if (response.isExists()) {
                            logger.info("missing id [{}] on shard {}", id, shardRouting);
                        }
                    }
                }
            }
        }
        //if there was an error we try to wait and see if at some point it'll get fixed
        logger.info("--> trying to wait");
        assertTrue(awaitBusy(() -> {
            boolean errorOccurred = false;
            for (int i = 0; i < iterations; i++) {
                SearchResponse searchResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get();
                if (searchResponse.getHits().getTotalHits() != numberOfDocs) {
                    errorOccurred = true;
                }
            }
            return !errorOccurred;
        }, 5, TimeUnit.MINUTES));
        assertEquals(numberOfDocs, ids.size());
    }
    //lets now make the test fail if it was supposed to fail
    for (int i = 0; i < iterations; i++) {
        assertHitCount(iterationResults[i], numberOfDocs);
    }
}

Example 14 with ShardStats

use of org.elasticsearch.action.admin.indices.stats.ShardStats in project elasticsearch by elastic.

the class RelocationIT method beforeIndexDeletion.

@Override
protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
        for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
                Optional<ShardStats> maybePrimary = Stream.of(indexShardStats.getShards()).filter(s -> s.getShardRouting().active() && s.getShardRouting().primary()).findFirst();
                if (maybePrimary.isPresent() == false) {
                    continue;
                }
                ShardStats primary = maybePrimary.get();
                final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
                assertThat(primary.getShardRouting() + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
                for (ShardStats shardStats : indexShardStats) {
                    final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                    assertThat(shardStats.getShardRouting() + " local checkpoint mismatch", seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " global checkpoint mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " max seq no mismatch", seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
                }
            }
        }
    });
}

Example 15 with ShardStats

use of org.elasticsearch.action.admin.indices.stats.ShardStats in project elasticsearch by elastic.

the class DiskUsageTests method testFillShardsWithShadowIndices.

public void testFillShardsWithShadowIndices() {
    final Index index = new Index("non-shadow", "0xcafe0000");
    ShardRouting s0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    s0 = ShardRoutingHelper.initialize(s0, "node1");
    s0 = ShardRoutingHelper.moveToStarted(s0);
    Path i0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
    CommonStats commonStats0 = new CommonStats();
    commonStats0.store = new StoreStats(100);
    final Index index2 = new Index("shadow", "0xcafe0001");
    ShardRouting s1 = ShardRouting.newUnassigned(new ShardId(index2, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    s1 = ShardRoutingHelper.initialize(s1, "node2");
    s1 = ShardRoutingHelper.moveToStarted(s1);
    Path i1Path = createTempDir().resolve("indices").resolve(index2.getUUID()).resolve("0");
    CommonStats commonStats1 = new CommonStats();
    commonStats1.store = new StoreStats(1000);
    ShardStats[] stats = new ShardStats[] { new ShardStats(s0, new ShardPath(false, i0Path, i0Path, s0.shardId()), commonStats0, null, null), new ShardStats(s1, new ShardPath(false, i1Path, i1Path, s1.shardId()), commonStats1, null, null) };
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
    ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).metaData(MetaData.builder().put(IndexMetaData.builder("non-shadow").settings(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xcafe0000").put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).put(IndexMetaData.builder("shadow").settings(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xcafe0001").put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))).build();
    logger.info("--> calling buildShardLevelInfo with state: {}", state);
    InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath, state);
    assertEquals(2, shardSizes.size());
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(s0)));
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(s1)));
    assertEquals(100L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(s0)).longValue());
    assertEquals(0L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(s1)).longValue());
}