use of org.opensearch.index.engine.SegmentsStats in project OpenSearch by opensearch-project.
the class SplitIndexIT method testCreateSplitIndex.
public void testCreateSplitIndex() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(2);
Version version = VersionUtils.randomIndexCompatibleVersion(random());
prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", 1).put("index.version.created", version)).get();
final int docs = randomIntBetween(0, 128);
for (int i = 0; i < docs; i++) {
client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
}
// ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
// if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
// to the require._name below.
ensureGreen();
// relocate all shards to one node such that we can merge it.
client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.blocks.write", true)).get();
ensureGreen();
final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
// disable rebalancing to be able to capture the right stats. balancing can move the target primary
// making it hard to pin point the source shards.
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")).get();
try {
final boolean createWithReplicas = randomBoolean();
assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setResizeType(ResizeType.SPLIT).setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).put("index.number_of_shards", 2).putNull("index.blocks.write").build()).get());
ensureGreen();
final ClusterState state = client().admin().cluster().prepareState().get().getState();
DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId());
logger.info("split node {}", mergeNode);
final long maxSeqNo = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getSeqNoStats).mapToLong(SeqNoStats::getMaxSeqNo).max().getAsLong();
final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getStats).map(CommonStats::getSegments).mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp).max().getAsLong();
final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
for (final ShardStats shardStats : targetStats.getShards()) {
final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
final ShardRouting shardRouting = shardStats.getShardRouting();
assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo));
assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo));
assertThat("failed on " + shardRouting, shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(), equalTo(maxUnsafeAutoIdTimestamp));
}
final int size = docs > 0 ? 2 * docs : 1;
assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
if (createWithReplicas == false) {
// bump replicas
client().admin().indices().prepareUpdateSettings("target").setSettings(Settings.builder().put("index.number_of_replicas", 1)).get();
ensureGreen();
assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
}
for (int i = docs; i < 2 * docs; i++) {
client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
}
flushAndRefresh();
assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 2 * docs);
assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
} finally {
// clean up
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)).get();
}
}
use of org.opensearch.index.engine.SegmentsStats in project OpenSearch by opensearch-project.
the class ShrinkIndexIT method testCreateShrinkIndex.
public void testCreateShrinkIndex() {
internalCluster().ensureAtLeastNumDataNodes(2);
Version version = VersionUtils.randomVersion(random());
prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("index.version.created", version)).get();
final int docs = randomIntBetween(0, 128);
for (int i = 0; i < docs; i++) {
client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
}
ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
// ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
// if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
// to the require._name below.
ensureGreen();
// relocate all shards to one node such that we can merge it.
client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.routing.allocation.require._name", discoveryNodes[0].getName()).put("index.blocks.write", true)).get();
ensureGreen();
final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
// disable rebalancing to be able to capture the right stats. balancing can move the target primary
// making it hard to pin point the source shards.
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")).get();
// now merge source into a single shard index
final boolean createWithReplicas = randomBoolean();
assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name").build()).get());
ensureGreen();
// resolve true merge node - this is not always the node we required as all shards may be on another node
final ClusterState state = client().admin().cluster().prepareState().get().getState();
DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId());
logger.info("merge node {}", mergeNode);
final long maxSeqNo = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getSeqNoStats).mapToLong(SeqNoStats::getMaxSeqNo).max().getAsLong();
final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getStats).map(CommonStats::getSegments).mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp).max().getAsLong();
final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
for (final ShardStats shardStats : targetStats.getShards()) {
final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
final ShardRouting shardRouting = shardStats.getShardRouting();
assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo));
assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo));
assertThat("failed on " + shardRouting, shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(), equalTo(maxUnsafeAutoIdTimestamp));
}
final int size = docs > 0 ? 2 * docs : 1;
assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
if (createWithReplicas == false) {
// bump replicas
client().admin().indices().prepareUpdateSettings("target").setSettings(Settings.builder().put("index.number_of_replicas", 1)).get();
ensureGreen();
assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
}
for (int i = docs; i < 2 * docs; i++) {
client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
}
flushAndRefresh();
assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 2 * docs);
assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
// clean up
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)).get();
}
use of org.opensearch.index.engine.SegmentsStats in project OpenSearch by opensearch-project.
the class CommonStats method add.
public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
docs = new DocsStats();
docs.add(stats.getDocs());
}
} else {
docs.add(stats.getDocs());
}
if (store == null) {
if (stats.getStore() != null) {
store = new StoreStats();
store.add(stats.getStore());
}
} else {
store.add(stats.getStore());
}
if (indexing == null) {
if (stats.getIndexing() != null) {
indexing = new IndexingStats();
indexing.add(stats.getIndexing());
}
} else {
indexing.add(stats.getIndexing());
}
if (get == null) {
if (stats.getGet() != null) {
get = new GetStats();
get.add(stats.getGet());
}
} else {
get.add(stats.getGet());
}
if (search == null) {
if (stats.getSearch() != null) {
search = new SearchStats();
search.add(stats.getSearch());
}
} else {
search.add(stats.getSearch());
}
if (merge == null) {
if (stats.getMerge() != null) {
merge = new MergeStats();
merge.add(stats.getMerge());
}
} else {
merge.add(stats.getMerge());
}
if (refresh == null) {
if (stats.getRefresh() != null) {
refresh = new RefreshStats();
refresh.add(stats.getRefresh());
}
} else {
refresh.add(stats.getRefresh());
}
if (flush == null) {
if (stats.getFlush() != null) {
flush = new FlushStats();
flush.add(stats.getFlush());
}
} else {
flush.add(stats.getFlush());
}
if (warmer == null) {
if (stats.getWarmer() != null) {
warmer = new WarmerStats();
warmer.add(stats.getWarmer());
}
} else {
warmer.add(stats.getWarmer());
}
if (queryCache == null) {
if (stats.getQueryCache() != null) {
queryCache = new QueryCacheStats();
queryCache.add(stats.getQueryCache());
}
} else {
queryCache.add(stats.getQueryCache());
}
if (fieldData == null) {
if (stats.getFieldData() != null) {
fieldData = new FieldDataStats();
fieldData.add(stats.getFieldData());
}
} else {
fieldData.add(stats.getFieldData());
}
if (completion == null) {
if (stats.getCompletion() != null) {
completion = new CompletionStats();
completion.add(stats.getCompletion());
}
} else {
completion.add(stats.getCompletion());
}
if (segments == null) {
if (stats.getSegments() != null) {
segments = new SegmentsStats();
segments.add(stats.getSegments());
}
} else {
segments.add(stats.getSegments());
}
if (translog == null) {
if (stats.getTranslog() != null) {
translog = new TranslogStats();
translog.add(stats.getTranslog());
}
} else {
translog.add(stats.getTranslog());
}
if (requestCache == null) {
if (stats.getRequestCache() != null) {
requestCache = new RequestCacheStats();
requestCache.add(stats.getRequestCache());
}
} else {
requestCache.add(stats.getRequestCache());
}
if (recoveryStats == null) {
if (stats.getRecoveryStats() != null) {
recoveryStats = new RecoveryStats();
recoveryStats.add(stats.getRecoveryStats());
}
} else {
recoveryStats.add(stats.getRecoveryStats());
}
}
use of org.opensearch.index.engine.SegmentsStats in project OpenSearch by opensearch-project.
the class RestShardsAction method buildTable.
// package private for testing
Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
Table table = getTableWithHeader(request);
for (ShardRouting shard : state.getState().routingTable().allShards()) {
ShardStats shardStats = stats.asMap().get(shard);
CommonStats commonStats = null;
CommitStats commitStats = null;
if (shardStats != null) {
commonStats = shardStats.getStats();
commitStats = shardStats.getCommitStats();
}
table.startRow();
table.addCell(shard.getIndexName());
table.addCell(shard.id());
if (shard.primary()) {
table.addCell("p");
} else {
table.addCell("r");
}
table.addCell(shard.state());
table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
if (shard.assignedToNode()) {
String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
String nodeId = shard.currentNodeId();
StringBuilder name = new StringBuilder();
name.append(state.getState().nodes().get(shard.currentNodeId()).getName());
if (shard.relocating()) {
String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress();
String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName();
String reloNodeId = shard.relocatingNodeId();
name.append(" -> ");
name.append(reloIp);
name.append(" ");
name.append(reloNodeId);
name.append(" ");
name.append(reloNme);
}
table.addCell(ip);
table.addCell(nodeId);
table.addCell(name);
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
if (shard.unassignedInfo() != null) {
table.addCell(shard.unassignedInfo().getReason());
Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
table.addCell(shard.unassignedInfo().getDetails());
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
if (shard.recoverySource() != null) {
table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
} else {
table.addCell(null);
}
table.addCell(getOrNull(commonStats, CommonStats::getCompletion, CompletionStats::getSize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingCount));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getListeners));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, SearchStats::getOpenContexts));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getIndexWriterMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getVersionMapMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getBitsetMemory));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getMaxSeqNo));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getLocalCheckpoint));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getGlobalCheckpoint));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
table.endRow();
}
return table;
}
use of org.opensearch.index.engine.SegmentsStats in project OpenSearch by opensearch-project.
the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.
/**
* Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
* failing. If auto generated ids are used this must not lead to duplicate ids
* see https://github.com/elastic/elasticsearch/issues/8788
*/
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
int numDocs = scaledRandomIntBetween(100, 1000);
Client client = internalCluster().coordOnlyNodeClient();
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
ensureGreen("index");
logger.info("unlucky node: {}", unluckyNode.getNode());
// create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
for (NodeStats dataNode : nodeStats.getNodes()) {
MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
mockTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), (connection, requestId, action, request, options) -> {
connection.sendRequest(requestId, action, request, options);
if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
logger.debug("Throw ConnectTransportException");
throw new ConnectTransportException(connection.getNode(), action);
}
});
}
BulkRequestBuilder bulkBuilder = client.prepareBulk();
for (int i = 0; i < numDocs; i++) {
XContentBuilder doc = null;
doc = jsonBuilder().startObject().field("foo", "bar").endObject();
bulkBuilder.add(client.prepareIndex("index").setSource(doc));
}
BulkResponse response = bulkBuilder.get();
if (response.hasFailures()) {
for (BulkItemResponse singleIndexRespons : response.getItems()) {
if (singleIndexRespons.isFailed()) {
fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
}
}
}
refresh();
SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
Set<String> uniqueIds = new HashSet<>();
long dupCounter = 0;
boolean found_duplicate_already = false;
for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
if (!found_duplicate_already) {
SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
assertThat(dupIdResponse.getHits().getTotalHits().value, greaterThan(1L));
logger.info("found a duplicate id:");
for (SearchHit hit : dupIdResponse.getHits()) {
logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
}
logger.info("will not print anymore in case more duplicates are found.");
found_duplicate_already = true;
}
dupCounter++;
}
}
assertSearchResponse(searchResponse);
assertThat(dupCounter, equalTo(0L));
assertHitCount(searchResponse, numDocs);
IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
IndexStats indexStats = index.getIndex("index");
long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
for (IndexShardStats indexShardStats : indexStats) {
for (ShardStats shardStats : indexShardStats) {
SegmentsStats segments = shardStats.getStats().getSegments();
maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
}
}
assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}
Aggregations