use of org.elasticsearch.action.admin.indices.stats.IndexStats in project elasticsearch by elastic.
the class FlushIT method testSyncedFlush.
public void testSyncedFlush() throws ExecutionException, InterruptedException, IOException {
internalCluster().ensureAtLeastNumDataNodes(2);
prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).get();
ensureGreen();
final Index index = client().admin().cluster().prepareState().get().getState().metaData().index("test").getIndex();
IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
for (ShardStats shardStats : indexStats.getShards()) {
assertNull(shardStats.getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
}
ShardsSyncedFlushResult result;
if (randomBoolean()) {
logger.info("--> sync flushing shard 0");
result = SyncedFlushUtil.attemptSyncedFlush(internalCluster(), new ShardId(index, 0));
} else {
logger.info("--> sync flushing index [test]");
SyncedFlushResponse indicesResult = client().admin().indices().prepareSyncedFlush("test").get();
result = indicesResult.getShardsResultPerIndex().get("test").get(0);
}
assertFalse(result.failed());
assertThat(result.totalShards(), equalTo(indexStats.getShards().length));
assertThat(result.successfulShards(), equalTo(indexStats.getShards().length));
indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
String syncId = result.syncId();
for (ShardStats shardStats : indexStats.getShards()) {
final String shardSyncId = shardStats.getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID);
assertThat(shardSyncId, equalTo(syncId));
}
// now, start new node and relocate a shard there and see if sync id still there
String newNodeName = internalCluster().startNode();
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
ShardRouting shardRouting = clusterState.getRoutingTable().index("test").shard(0).iterator().next();
String currentNodeName = clusterState.nodes().resolveNode(shardRouting.currentNodeId()).getName();
assertFalse(currentNodeName.equals(newNodeName));
internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, currentNodeName, newNodeName)).get();
client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).get();
indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
for (ShardStats shardStats : indexStats.getShards()) {
assertNotNull(shardStats.getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
}
client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build()).get();
ensureGreen("test");
indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
for (ShardStats shardStats : indexStats.getShards()) {
assertNotNull(shardStats.getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
}
client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, internalCluster().numDataNodes() - 1).build()).get();
ensureGreen("test");
indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
for (ShardStats shardStats : indexStats.getShards()) {
assertNotNull(shardStats.getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
}
}
use of org.elasticsearch.action.admin.indices.stats.IndexStats in project elasticsearch by elastic.
the class RestIndicesAction method buildTable.
// package private for testing
Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse response, IndicesStatsResponse stats, MetaData indexMetaDatas) {
final String healthParam = request.param("health");
final ClusterHealthStatus status;
if (healthParam != null) {
status = ClusterHealthStatus.fromString(healthParam);
} else {
status = null;
}
Table table = getTableWithHeader(request);
for (final Index index : indices) {
final String indexName = index.getName();
ClusterIndexHealth indexHealth = response.getIndices().get(indexName);
IndexStats indexStats = stats.getIndices().get(indexName);
IndexMetaData indexMetaData = indexMetaDatas.getIndices().get(indexName);
IndexMetaData.State state = indexMetaData.getState();
if (status != null) {
if (state == IndexMetaData.State.CLOSE || (indexHealth == null && !ClusterHealthStatus.RED.equals(status)) || !indexHealth.getStatus().equals(status)) {
continue;
}
}
table.startRow();
table.addCell(state == IndexMetaData.State.OPEN ? (indexHealth == null ? "red*" : indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)) : null);
table.addCell(state.toString().toLowerCase(Locale.ROOT));
table.addCell(indexName);
table.addCell(index.getUUID());
table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards());
table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getDeleted());
table.addCell(indexMetaData.getCreationDate());
table.addCell(new DateTime(indexMetaData.getCreationDate(), DateTimeZone.UTC));
table.addCell(indexStats == null ? null : indexStats.getTotal().getStore().size());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getStore().size());
table.addCell(indexStats == null ? null : indexStats.getTotal().getCompletion().getSize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getCompletion().getSize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getFieldData().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFieldData().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getFieldData().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFieldData().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getTotal().getQueryCache().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getQueryCache().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getQueryCache().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getQueryCache().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getMemorySize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getEvictions());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getHitCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getHitCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRequestCache().getMissCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRequestCache().getMissCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getFlush().getTotal());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFlush().getTotal());
table.addCell(indexStats == null ? null : indexStats.getTotal().getFlush().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getFlush().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().current());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().current());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getExistsTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getExistsTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getExistsCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getExistsCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getMissingTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getMissingTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getGet().getMissingCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getGet().getMissingCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getDeleteCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getDeleteCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexFailedCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexFailedCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentNumDocs());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentNumDocs());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentSize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotal());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotal());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalNumDocs());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalNumDocs());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalSize());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalSize());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotal());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotal());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotalTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getListeners());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getListeners());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getOpenContexts());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getOpenContexts());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getQueryCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getQueryCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getScrollCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getScrollCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getBitsetMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getBitsetMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().current());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().current());
table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().total());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().total());
table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().totalTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().totalTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCurrent());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestTime());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestTime());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getTotalMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getTotalMemory());
table.endRow();
}
return table;
}
use of org.elasticsearch.action.admin.indices.stats.IndexStats in project elasticsearch by elastic.
the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.
/**
* Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
* failing. If auto generated ids are used this must not lead to duplicate ids
* see https://github.com/elastic/elasticsearch/issues/8788
*/
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
int numDocs = scaledRandomIntBetween(100, 1000);
Client client = internalCluster().coordOnlyNodeClient();
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
ensureGreen("index");
logger.info("unlucky node: {}", unluckyNode.getNode());
//create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
for (NodeStats dataNode : nodeStats.getNodes()) {
MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {
@Override
protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
super.sendRequest(connection, requestId, action, request, options);
if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
logger.debug("Throw ConnectTransportException");
throw new ConnectTransportException(connection.getNode(), action);
}
}
});
}
BulkRequestBuilder bulkBuilder = client.prepareBulk();
for (int i = 0; i < numDocs; i++) {
XContentBuilder doc = null;
doc = jsonBuilder().startObject().field("foo", "bar").endObject();
bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
}
BulkResponse response = bulkBuilder.get();
if (response.hasFailures()) {
for (BulkItemResponse singleIndexRespons : response.getItems()) {
if (singleIndexRespons.isFailed()) {
fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
}
}
}
refresh();
SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
Set<String> uniqueIds = new HashSet();
long dupCounter = 0;
boolean found_duplicate_already = false;
for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
if (!found_duplicate_already) {
SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
logger.info("found a duplicate id:");
for (SearchHit hit : dupIdResponse.getHits()) {
logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
}
logger.info("will not print anymore in case more duplicates are found.");
found_duplicate_already = true;
}
dupCounter++;
}
}
assertSearchResponse(searchResponse);
assertThat(dupCounter, equalTo(0L));
assertHitCount(searchResponse, numDocs);
IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
IndexStats indexStats = index.getIndex("index");
long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
for (IndexShardStats indexShardStats : indexStats) {
for (ShardStats shardStats : indexShardStats) {
SegmentsStats segments = shardStats.getStats().getSegments();
maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
}
}
assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}
use of org.elasticsearch.action.admin.indices.stats.IndexStats in project elasticsearch by elastic.
the class IndexShardIT method testMarkAsInactiveTriggersSyncedFlush.
public void testMarkAsInactiveTriggersSyncedFlush() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
ensureGreen("test");
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
assertBusy(() -> {
IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test");
assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
});
IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
}
use of org.elasticsearch.action.admin.indices.stats.IndexStats in project graylog2-server by Graylog2.
the class Indices method getIndicesStats.
public Set<IndexStatistics> getIndicesStats(final IndexSet indexSet) {
final Map<String, IndexStats> responseIndices;
try {
responseIndices = getAll(indexSet);
} catch (ElasticsearchException e) {
return Collections.emptySet();
}
final ImmutableSet.Builder<IndexStatistics> result = ImmutableSet.builder();
for (IndexStats indexStats : responseIndices.values()) {
final ImmutableList.Builder<ShardRouting> shardRouting = ImmutableList.builder();
for (ShardStats shardStats : indexStats.getShards()) {
shardRouting.add(shardStats.getShardRouting());
}
final IndexStatistics stats = IndexStatistics.create(indexStats.getIndex(), indexStats.getPrimaries(), indexStats.getTotal(), shardRouting.build());
result.add(stats);
}
return result.build();
}
Aggregations