Examples with IndexShardStats org.elasticsearch.action.admin.indices.stats.IndexShardStats used on opensource projects

Example 1 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class NodeIndicesStats method readFrom.

@Override
public void readFrom(StreamInput in) throws IOException {
    stats = new CommonStats(in);
    if (in.readBoolean()) {
        int entries = in.readVInt();
        statsByShard = new HashMap<>();
        for (int i = 0; i < entries; i++) {
            Index index = new Index(in);
            int indexShardListSize = in.readVInt();
            List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
            for (int j = 0; j < indexShardListSize; j++) {
                indexShardStats.add(IndexShardStats.readIndexShardStats(in));
            }
            statsByShard.put(index, indexShardStats);
        }
    }
}

Example 2 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 3 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class IndicesService method stats.

public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
    CommonStats oldStats = new CommonStats(flags);
    if (includePrevious) {
        Flag[] setFlags = flags.getFlags();
        for (Flag flag : setFlags) {
            switch(flag) {
                case Get:
                    oldStats.get.add(oldShardsStats.getStats);
                    break;
                case Indexing:
                    oldStats.indexing.add(oldShardsStats.indexingStats);
                    break;
                case Search:
                    oldStats.search.add(oldShardsStats.searchStats);
                    break;
                case Merge:
                    oldStats.merge.add(oldShardsStats.mergeStats);
                    break;
                case Refresh:
                    oldStats.refresh.add(oldShardsStats.refreshStats);
                    break;
                case Recovery:
                    oldStats.recoveryStats.add(oldShardsStats.recoveryStats);
                    break;
                case Flush:
                    oldStats.flush.add(oldShardsStats.flushStats);
                    break;
            }
        }
    }
    Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
    for (IndexService indexService : this) {
        for (IndexShard indexShard : indexService) {
            try {
                if (indexShard.routingEntry() == null) {
                    continue;
                }
                IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats(), indexShard.seqNoStats()) });
                if (!statsByShard.containsKey(indexService.index())) {
                    statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                } else {
                    statsByShard.get(indexService.index()).add(indexShardStats);
                }
            } catch (IllegalIndexShardStateException e) {
                // we can safely ignore illegal state on ones that are closing for example
                logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} ignoring shard stats", indexShard.shardId()), e);
            }
        }
    }
    return new NodeIndicesStats(oldStats, statsByShard);
}

Example 4 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class NodeIndicesStats method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    stats.writeTo(out);
    out.writeBoolean(statsByShard != null);
    if (statsByShard != null) {
        out.writeVInt(statsByShard.size());
        for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
            entry.getKey().writeTo(out);
            out.writeVInt(entry.getValue().size());
            for (IndexShardStats indexShardStats : entry.getValue()) {
                indexShardStats.writeTo(out);
            }
        }
    }
}

Example 5 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class RelocationIT method beforeIndexDeletion.

@Override
protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
        for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
                Optional<ShardStats> maybePrimary = Stream.of(indexShardStats.getShards()).filter(s -> s.getShardRouting().active() && s.getShardRouting().primary()).findFirst();
                if (maybePrimary.isPresent() == false) {
                    continue;
                }
                ShardStats primary = maybePrimary.get();
                final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
                assertThat(primary.getShardRouting() + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
                for (ShardStats shardStats : indexShardStats) {
                    final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                    assertThat(shardStats.getShardRouting() + " local checkpoint mismatch", seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " global checkpoint mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " max seq no mismatch", seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
                }
            }
        }
    });
}