Search in sources :

Example 1 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class NodeIndicesStats method readFrom.

@Override
public void readFrom(StreamInput in) throws IOException {
    stats = new CommonStats(in);
    if (in.readBoolean()) {
        int entries = in.readVInt();
        statsByShard = new HashMap<>();
        for (int i = 0; i < entries; i++) {
            Index index = new Index(in);
            int indexShardListSize = in.readVInt();
            List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
            for (int j = 0; j < indexShardListSize; j++) {
                indexShardStats.add(IndexShardStats.readIndexShardStats(in));
            }
            statsByShard.put(index, indexShardStats);
        }
    }
}
Also used : CommonStats(org.elasticsearch.action.admin.indices.stats.CommonStats) ArrayList(java.util.ArrayList) Index(org.elasticsearch.index.Index) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats)

Example 2 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}
Also used : MockTransportService(org.elasticsearch.test.transport.MockTransportService) SearchHit(org.elasticsearch.search.SearchHit) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) SegmentsStats(org.elasticsearch.index.engine.SegmentsStats) NodesStatsResponse(org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse) NodeStats(org.elasticsearch.action.admin.cluster.node.stats.NodeStats) TransportRequestOptions(org.elasticsearch.transport.TransportRequestOptions) Client(org.elasticsearch.client.Client) HashSet(java.util.HashSet) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) ShardStats(org.elasticsearch.action.admin.indices.stats.ShardStats) IndicesStatsResponse(org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse) TransportRequest(org.elasticsearch.transport.TransportRequest) BulkItemResponse(org.elasticsearch.action.bulk.BulkItemResponse) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) IOException(java.io.IOException) SearchResponse(org.elasticsearch.action.search.SearchResponse) ElasticsearchAssertions.assertSearchResponse(org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MockTransportService(org.elasticsearch.test.transport.MockTransportService) TransportService(org.elasticsearch.transport.TransportService) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) BulkRequestBuilder(org.elasticsearch.action.bulk.BulkRequestBuilder) IndexStats(org.elasticsearch.action.admin.indices.stats.IndexStats) XContentBuilder(org.elasticsearch.common.xcontent.XContentBuilder)

Example 3 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class IndicesService method stats.

public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
    CommonStats oldStats = new CommonStats(flags);
    if (includePrevious) {
        Flag[] setFlags = flags.getFlags();
        for (Flag flag : setFlags) {
            switch(flag) {
                case Get:
                    oldStats.get.add(oldShardsStats.getStats);
                    break;
                case Indexing:
                    oldStats.indexing.add(oldShardsStats.indexingStats);
                    break;
                case Search:
                    oldStats.search.add(oldShardsStats.searchStats);
                    break;
                case Merge:
                    oldStats.merge.add(oldShardsStats.mergeStats);
                    break;
                case Refresh:
                    oldStats.refresh.add(oldShardsStats.refreshStats);
                    break;
                case Recovery:
                    oldStats.recoveryStats.add(oldShardsStats.recoveryStats);
                    break;
                case Flush:
                    oldStats.flush.add(oldShardsStats.flushStats);
                    break;
            }
        }
    }
    Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
    for (IndexService indexService : this) {
        for (IndexShard indexShard : indexService) {
            try {
                if (indexShard.routingEntry() == null) {
                    continue;
                }
                IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats(), indexShard.seqNoStats()) });
                if (!statsByShard.containsKey(indexService.index())) {
                    statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                } else {
                    statsByShard.get(indexService.index()).add(indexShardStats);
                }
            } catch (IllegalIndexShardStateException e) {
                // we can safely ignore illegal state on ones that are closing for example
                logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} ignoring shard stats", indexShard.shardId()), e);
            }
        }
    }
    return new NodeIndicesStats(oldStats, statsByShard);
}
Also used : IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) ShardStats(org.elasticsearch.action.admin.indices.stats.ShardStats) HashMap(java.util.HashMap) IndexService(org.elasticsearch.index.IndexService) IndexShard(org.elasticsearch.index.shard.IndexShard) Index(org.elasticsearch.index.Index) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) IllegalIndexShardStateException(org.elasticsearch.index.shard.IllegalIndexShardStateException) Flag(org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag) CommonStats(org.elasticsearch.action.admin.indices.stats.CommonStats) CollectionUtils.arrayAsArrayList(org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList) ArrayList(java.util.ArrayList) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List) Supplier(java.util.function.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage)

Example 4 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class NodeIndicesStats method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    stats.writeTo(out);
    out.writeBoolean(statsByShard != null);
    if (statsByShard != null) {
        out.writeVInt(statsByShard.size());
        for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
            entry.getKey().writeTo(out);
            out.writeVInt(entry.getValue().size());
            for (IndexShardStats indexShardStats : entry.getValue()) {
                indexShardStats.writeTo(out);
            }
        }
    }
}
Also used : Index(org.elasticsearch.index.Index) ArrayList(java.util.ArrayList) List(java.util.List) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) HashMap(java.util.HashMap) Map(java.util.Map)

Example 5 with IndexShardStats

use of org.elasticsearch.action.admin.indices.stats.IndexShardStats in project elasticsearch by elastic.

the class RelocationIT method beforeIndexDeletion.

@Override
protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
        for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
                Optional<ShardStats> maybePrimary = Stream.of(indexShardStats.getShards()).filter(s -> s.getShardRouting().active() && s.getShardRouting().primary()).findFirst();
                if (maybePrimary.isPresent() == false) {
                    continue;
                }
                ShardStats primary = maybePrimary.get();
                final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
                assertThat(primary.getShardRouting() + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
                for (ShardStats shardStats : indexShardStats) {
                    final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                    assertThat(shardStats.getShardRouting() + " local checkpoint mismatch", seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " global checkpoint mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " max seq no mismatch", seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
                }
            }
        }
    });
}
Also used : ShardId(org.elasticsearch.index.shard.ShardId) Arrays(java.util.Arrays) TransportRequest(org.elasticsearch.transport.TransportRequest) Nullable(org.elasticsearch.common.Nullable) SearchHits(org.elasticsearch.search.SearchHits) Matchers.not(org.hamcrest.Matchers.not) IndexStats(org.elasticsearch.action.admin.indices.stats.IndexStats) MoveAllocationCommand(org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand) SeqNoStats(org.elasticsearch.index.seqno.SeqNoStats) ClusterState(org.elasticsearch.cluster.ClusterState) Settings(org.elasticsearch.common.settings.Settings) Scope(org.elasticsearch.test.ESIntegTestCase.Scope) SearchResponse(org.elasticsearch.action.search.SearchResponse) Path(java.nio.file.Path) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) SearchHit(org.elasticsearch.search.SearchHit) MockIndexEventListener(org.elasticsearch.test.MockIndexEventListener) Priority(org.elasticsearch.common.Priority) PeerRecoveryTargetService(org.elasticsearch.indices.recovery.PeerRecoveryTargetService) Transport(org.elasticsearch.transport.Transport) TestLogging(org.elasticsearch.test.junit.annotations.TestLogging) Collection(java.util.Collection) RecoveryFileChunkRequest(org.elasticsearch.indices.recovery.RecoveryFileChunkRequest) ElasticsearchAssertions.assertHitCount(org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount) Matchers.startsWith(org.hamcrest.Matchers.startsWith) ElasticsearchAssertions.assertSearchHits(org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits) FileVisitResult(java.nio.file.FileVisitResult) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stream(java.util.stream.Stream) IndexRequestBuilder(org.elasticsearch.action.index.IndexRequestBuilder) ESIntegTestCase(org.elasticsearch.test.ESIntegTestCase) IndexMetaData(org.elasticsearch.cluster.metadata.IndexMetaData) INDEX_SEQ_NO_CHECKPOINT_SYNC_INTERVAL(org.elasticsearch.index.IndexSettings.INDEX_SEQ_NO_CHECKPOINT_SYNC_INTERVAL) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Optional(java.util.Optional) TransportRequestOptions(org.elasticsearch.transport.TransportRequestOptions) IntProcedure(com.carrotsearch.hppc.procedures.IntProcedure) XContentType(org.elasticsearch.common.xcontent.XContentType) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) ClusterService(org.elasticsearch.cluster.service.ClusterService) ShardRoutingState(org.elasticsearch.cluster.routing.ShardRoutingState) ArrayList(java.util.ArrayList) IndicesStatsResponse(org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) SequenceNumbersService(org.elasticsearch.index.seqno.SequenceNumbersService) ShardStats(org.elasticsearch.action.admin.indices.stats.ShardStats) TimeValue(org.elasticsearch.common.unit.TimeValue) MockTransportService(org.elasticsearch.test.transport.MockTransportService) TransportService(org.elasticsearch.transport.TransportService) EnableAllocationDecider(org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider) ClusterScope(org.elasticsearch.test.ESIntegTestCase.ClusterScope) IndexShardState(org.elasticsearch.index.shard.IndexShardState) QueryBuilders.matchAllQuery(org.elasticsearch.index.query.QueryBuilders.matchAllQuery) IndexEventListener(org.elasticsearch.index.shard.IndexEventListener) Files(java.nio.file.Files) Semaphore(java.util.concurrent.Semaphore) Client(org.elasticsearch.client.Client) IndexShard(org.elasticsearch.index.shard.IndexShard) IntHashSet(com.carrotsearch.hppc.IntHashSet) IndexFileNames(org.apache.lucene.index.IndexFileNames) Plugin(org.elasticsearch.plugins.Plugin) BackgroundIndexer(org.elasticsearch.test.BackgroundIndexer) ClusterHealthResponse(org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse) IOException(java.io.IOException) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) NodeEnvironment(org.elasticsearch.env.NodeEnvironment) English(org.apache.lucene.util.English) ElasticsearchAssertions.assertAcked(org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked) ElasticsearchAssertions.assertNoFailures(org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) ShardStats(org.elasticsearch.action.admin.indices.stats.ShardStats) IndicesStatsResponse(org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse) SeqNoStats(org.elasticsearch.index.seqno.SeqNoStats) IndexShardStats(org.elasticsearch.action.admin.indices.stats.IndexShardStats) IndexStats(org.elasticsearch.action.admin.indices.stats.IndexStats)

Aggregations

IndexShardStats (org.elasticsearch.action.admin.indices.stats.IndexShardStats)5 ArrayList (java.util.ArrayList)4 List (java.util.List)3 ShardStats (org.elasticsearch.action.admin.indices.stats.ShardStats)3 Index (org.elasticsearch.index.Index)3 IOException (java.io.IOException)2 HashMap (java.util.HashMap)2 CommonStats (org.elasticsearch.action.admin.indices.stats.CommonStats)2 IndexStats (org.elasticsearch.action.admin.indices.stats.IndexStats)2 IndicesStatsResponse (org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse)2 SearchResponse (org.elasticsearch.action.search.SearchResponse)2 Client (org.elasticsearch.client.Client)2 IndexShard (org.elasticsearch.index.shard.IndexShard)2 IntHashSet (com.carrotsearch.hppc.IntHashSet)1 IntProcedure (com.carrotsearch.hppc.procedures.IntProcedure)1 FileVisitResult (java.nio.file.FileVisitResult)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 SimpleFileVisitor (java.nio.file.SimpleFileVisitor)1 BasicFileAttributes (java.nio.file.attribute.BasicFileAttributes)1