Examples with IndexShardStats org.opensearch.action.admin.indices.stats.IndexShardStats used on opensource projects

Example 1 with IndexShardStats

use of org.opensearch.action.admin.indices.stats.IndexShardStats in project OpenSearch by opensearch-project.

the class IndicesServiceTests method testStatsByShardDoesNotDieFromExpectedExceptions.

public void testStatsByShardDoesNotDieFromExpectedExceptions() {
    final int shardCount = randomIntBetween(2, 5);
    final int failedShardId = randomIntBetween(0, shardCount - 1);
    final Index index = new Index("test-index", "abc123");
    // the shard that is going to fail
    final ShardId shardId = new ShardId(index, failedShardId);
    final List<IndexShard> shards = new ArrayList<>(shardCount);
    final List<IndexShardStats> shardStats = new ArrayList<>(shardCount - 1);
    final IndexShardState state = randomFrom(IndexShardState.values());
    final String message = "TEST - expected";
    final RuntimeException expectedException = randomFrom(new IllegalIndexShardStateException(shardId, state, message), new AlreadyClosedException(message));
    // this allows us to control the indices that exist
    final IndicesService mockIndicesService = mock(IndicesService.class);
    final IndexService indexService = mock(IndexService.class);
    // generate fake shards and their responses
    for (int i = 0; i < shardCount; ++i) {
        final IndexShard shard = mock(IndexShard.class);
        shards.add(shard);
        if (failedShardId != i) {
            final IndexShardStats successfulShardStats = mock(IndexShardStats.class);
            shardStats.add(successfulShardStats);
            when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenReturn(successfulShardStats);
        } else {
            when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenThrow(expectedException);
        }
    }
    when(mockIndicesService.iterator()).thenReturn(Collections.singleton(indexService).iterator());
    when(indexService.iterator()).thenReturn(shards.iterator());
    when(indexService.index()).thenReturn(index);
    // real one, which has a logger defined
    final IndicesService indicesService = getIndicesService();
    final Map<Index, List<IndexShardStats>> indexStats = indicesService.statsByShard(mockIndicesService, CommonStatsFlags.ALL);
    assertThat(indexStats.isEmpty(), equalTo(false));
    assertThat("index not defined", indexStats.containsKey(index), equalTo(true));
    assertThat("unexpected shard stats", indexStats.get(index), equalTo(shardStats));
}

Example 2 with IndexShardStats

use of org.opensearch.action.admin.indices.stats.IndexShardStats in project OpenSearch by opensearch-project.

the class IndicesService method indexShardStats.

IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) {
    if (indexShard.routingEntry() == null) {
        return null;
    }
    CommitStats commitStats;
    SeqNoStats seqNoStats;
    RetentionLeaseStats retentionLeaseStats;
    try {
        commitStats = indexShard.commitStats();
        seqNoStats = indexShard.seqNoStats();
        retentionLeaseStats = indexShard.getRetentionLeaseStats();
    } catch (AlreadyClosedException e) {
        // shard is closed - no stats is fine
        commitStats = null;
        seqNoStats = null;
        retentionLeaseStats = null;
    }
    return new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), commitStats, seqNoStats, retentionLeaseStats) });
}

Example 3 with IndexShardStats

use of org.opensearch.action.admin.indices.stats.IndexShardStats in project OpenSearch by opensearch-project.

the class NodeIndicesStats method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    stats.writeTo(out);
    out.writeBoolean(statsByShard != null);
    if (statsByShard != null) {
        out.writeVInt(statsByShard.size());
        for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
            entry.getKey().writeTo(out);
            out.writeVInt(entry.getValue().size());
            for (IndexShardStats indexShardStats : entry.getValue()) {
                indexShardStats.writeTo(out);
            }
        }
    }
}

Example 4 with IndexShardStats

use of org.opensearch.action.admin.indices.stats.IndexShardStats in project OpenSearch by opensearch-project.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
 * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
 * failing. If auto generated ids are used this must not lead to duplicate ids
 * see https://github.com/elastic/elasticsearch/issues/8788
 */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    // create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> {
            connection.sendRequest(requestId, action, request, options);
            if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                logger.debug("Throw ConnectTransportException");
                throw new ConnectTransportException(connection.getNode(), action);
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet<>();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits().value, greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}