use of org.opensearch.index.flush.FlushStats in project OpenSearch by opensearch-project.
the class IndexShardIT method testMaybeFlush.
public void testMaybeFlush() throws Exception {
createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
ensureGreen();
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService(resolveIndex("test"));
IndexShard shard = test.getShardOrNull(0);
assertFalse(shard.shouldPeriodicallyFlush());
client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(135, /* size of the operation + one generation header&footer*/
ByteSizeUnit.BYTES)).build()).get();
client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
assertFalse(shard.shouldPeriodicallyFlush());
shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse("test", "_doc", "1", new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
assertTrue(shard.shouldPeriodicallyFlush());
final Translog translog = getTranslog(shard);
assertEquals(2, translog.stats().getUncommittedOperations());
assertThat(shard.flushStats().getTotal(), equalTo(0L));
client().prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
assertThat(shard.getLastKnownGlobalCheckpoint(), equalTo(2L));
assertBusy(() -> {
// this is async
assertFalse(shard.shouldPeriodicallyFlush());
assertThat(shard.flushStats().getPeriodic(), equalTo(1L));
assertThat(shard.flushStats().getTotal(), equalTo(1L));
});
shard.sync();
assertThat(shard.getLastSyncedGlobalCheckpoint(), equalTo(2L));
assertThat("last commit [" + shard.commitStats().getUserData() + "]", translog.stats().getUncommittedOperations(), equalTo(0));
long size = Math.max(translog.stats().getUncommittedSizeInBytes(), Translog.DEFAULT_HEADER_SIZE_IN_BYTES + 1);
logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)).build()).get();
client().prepareDelete("test", "2").get();
logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
assertBusy(() -> {
// this is async
final TranslogStats translogStats = translog.stats();
final CommitStats commitStats = shard.commitStats();
final FlushStats flushStats = shard.flushStats();
logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal());
assertFalse(shard.shouldPeriodicallyFlush());
});
shard.sync();
assertEquals(0, translog.stats().getUncommittedOperations());
}
use of org.opensearch.index.flush.FlushStats in project OpenSearch by opensearch-project.
the class IndexShardIT method testStressMaybeFlushOrRollTranslogGeneration.
public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception {
createIndex("test");
ensureGreen();
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService(resolveIndex("test"));
final IndexShard shard = test.getShardOrNull(0);
assertFalse(shard.shouldPeriodicallyFlush());
final boolean flush = randomBoolean();
final Settings settings;
if (flush) {
// size of the operation plus the overhead of one generation.
settings = Settings.builder().put("index.translog.flush_threshold_size", "125b").build();
} else {
// size of the operation plus header and footer
settings = Settings.builder().put("index.translog.generation_threshold_size", "117b").build();
}
client().admin().indices().prepareUpdateSettings("test").setSettings(settings).get();
client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
assertFalse(shard.shouldPeriodicallyFlush());
final AtomicBoolean running = new AtomicBoolean(true);
final int numThreads = randomIntBetween(2, 4);
final Thread[] threads = new Thread[numThreads];
final CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(() -> {
try {
barrier.await();
} catch (final InterruptedException | BrokenBarrierException e) {
throw new RuntimeException(e);
}
while (running.get()) {
shard.afterWriteOperation();
}
});
threads[i].start();
}
barrier.await();
final CheckedRunnable<Exception> check;
if (flush) {
final FlushStats initialStats = shard.flushStats();
client().prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get();
check = () -> {
assertFalse(shard.shouldPeriodicallyFlush());
final FlushStats currentStats = shard.flushStats();
String msg = String.format(Locale.ROOT, "flush stats: total=[%d vs %d], periodic=[%d vs %d]", initialStats.getTotal(), currentStats.getTotal(), initialStats.getPeriodic(), currentStats.getPeriodic());
assertThat(msg, currentStats.getPeriodic(), either(equalTo(initialStats.getPeriodic() + 1)).or(equalTo(initialStats.getPeriodic() + 2)));
assertThat(msg, currentStats.getTotal(), either(equalTo(initialStats.getTotal() + 1)).or(equalTo(initialStats.getTotal() + 2)));
};
} else {
final long generation = getTranslog(shard).currentFileGeneration();
client().prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get();
check = () -> {
assertFalse(shard.shouldRollTranslogGeneration());
assertEquals(generation + 1, getTranslog(shard).currentFileGeneration());
};
}
assertBusy(check);
running.set(false);
for (int i = 0; i < threads.length; i++) {
threads[i].join();
}
check.run();
}
use of org.opensearch.index.flush.FlushStats in project OpenSearch by opensearch-project.
the class CommonStats method add.
public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
docs = new DocsStats();
docs.add(stats.getDocs());
}
} else {
docs.add(stats.getDocs());
}
if (store == null) {
if (stats.getStore() != null) {
store = new StoreStats();
store.add(stats.getStore());
}
} else {
store.add(stats.getStore());
}
if (indexing == null) {
if (stats.getIndexing() != null) {
indexing = new IndexingStats();
indexing.add(stats.getIndexing());
}
} else {
indexing.add(stats.getIndexing());
}
if (get == null) {
if (stats.getGet() != null) {
get = new GetStats();
get.add(stats.getGet());
}
} else {
get.add(stats.getGet());
}
if (search == null) {
if (stats.getSearch() != null) {
search = new SearchStats();
search.add(stats.getSearch());
}
} else {
search.add(stats.getSearch());
}
if (merge == null) {
if (stats.getMerge() != null) {
merge = new MergeStats();
merge.add(stats.getMerge());
}
} else {
merge.add(stats.getMerge());
}
if (refresh == null) {
if (stats.getRefresh() != null) {
refresh = new RefreshStats();
refresh.add(stats.getRefresh());
}
} else {
refresh.add(stats.getRefresh());
}
if (flush == null) {
if (stats.getFlush() != null) {
flush = new FlushStats();
flush.add(stats.getFlush());
}
} else {
flush.add(stats.getFlush());
}
if (warmer == null) {
if (stats.getWarmer() != null) {
warmer = new WarmerStats();
warmer.add(stats.getWarmer());
}
} else {
warmer.add(stats.getWarmer());
}
if (queryCache == null) {
if (stats.getQueryCache() != null) {
queryCache = new QueryCacheStats();
queryCache.add(stats.getQueryCache());
}
} else {
queryCache.add(stats.getQueryCache());
}
if (fieldData == null) {
if (stats.getFieldData() != null) {
fieldData = new FieldDataStats();
fieldData.add(stats.getFieldData());
}
} else {
fieldData.add(stats.getFieldData());
}
if (completion == null) {
if (stats.getCompletion() != null) {
completion = new CompletionStats();
completion.add(stats.getCompletion());
}
} else {
completion.add(stats.getCompletion());
}
if (segments == null) {
if (stats.getSegments() != null) {
segments = new SegmentsStats();
segments.add(stats.getSegments());
}
} else {
segments.add(stats.getSegments());
}
if (translog == null) {
if (stats.getTranslog() != null) {
translog = new TranslogStats();
translog.add(stats.getTranslog());
}
} else {
translog.add(stats.getTranslog());
}
if (requestCache == null) {
if (stats.getRequestCache() != null) {
requestCache = new RequestCacheStats();
requestCache.add(stats.getRequestCache());
}
} else {
requestCache.add(stats.getRequestCache());
}
if (recoveryStats == null) {
if (stats.getRecoveryStats() != null) {
recoveryStats = new RecoveryStats();
recoveryStats.add(stats.getRecoveryStats());
}
} else {
recoveryStats.add(stats.getRecoveryStats());
}
}
use of org.opensearch.index.flush.FlushStats in project OpenSearch by opensearch-project.
the class RestShardsAction method buildTable.
// package private for testing
Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
Table table = getTableWithHeader(request);
for (ShardRouting shard : state.getState().routingTable().allShards()) {
ShardStats shardStats = stats.asMap().get(shard);
CommonStats commonStats = null;
CommitStats commitStats = null;
if (shardStats != null) {
commonStats = shardStats.getStats();
commitStats = shardStats.getCommitStats();
}
table.startRow();
table.addCell(shard.getIndexName());
table.addCell(shard.id());
if (shard.primary()) {
table.addCell("p");
} else {
table.addCell("r");
}
table.addCell(shard.state());
table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
if (shard.assignedToNode()) {
String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
String nodeId = shard.currentNodeId();
StringBuilder name = new StringBuilder();
name.append(state.getState().nodes().get(shard.currentNodeId()).getName());
if (shard.relocating()) {
String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress();
String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName();
String reloNodeId = shard.relocatingNodeId();
name.append(" -> ");
name.append(reloIp);
name.append(" ");
name.append(reloNodeId);
name.append(" ");
name.append(reloNme);
}
table.addCell(ip);
table.addCell(nodeId);
table.addCell(name);
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
if (shard.unassignedInfo() != null) {
table.addCell(shard.unassignedInfo().getReason());
Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
table.addCell(shard.unassignedInfo().getDetails());
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
if (shard.recoverySource() != null) {
table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
} else {
table.addCell(null);
}
table.addCell(getOrNull(commonStats, CommonStats::getCompletion, CompletionStats::getSize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingCount));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getListeners));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, SearchStats::getOpenContexts));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getIndexWriterMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getVersionMapMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getBitsetMemory));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getMaxSeqNo));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getLocalCheckpoint));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getGlobalCheckpoint));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
table.endRow();
}
return table;
}
use of org.opensearch.index.flush.FlushStats in project OpenSearch by opensearch-project.
the class IndexShardIT method testFlushStats.
public void testFlushStats() throws Exception {
final IndexService indexService = createIndex("test");
ensureGreen();
Settings settings = Settings.builder().put("index.translog.flush_threshold_size", "" + between(200, 300) + "b").build();
client().admin().indices().prepareUpdateSettings("test").setSettings(settings).get();
final int numDocs = between(10, 100);
for (int i = 0; i < numDocs; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("{}", XContentType.JSON).get();
}
// A flush stats may include the new total count but the old period count - assert eventually.
assertBusy(() -> {
final FlushStats flushStats = client().admin().indices().prepareStats("test").clear().setFlush(true).get().getTotal().flush;
assertThat(flushStats.getPeriodic(), allOf(equalTo(flushStats.getTotal()), greaterThan(0L)));
});
assertBusy(() -> assertThat(indexService.getShard(0).shouldPeriodicallyFlush(), equalTo(false)));
settings = Settings.builder().put("index.translog.flush_threshold_size", (String) null).build();
client().admin().indices().prepareUpdateSettings("test").setSettings(settings).get();
client().prepareIndex("test").setId(UUIDs.randomBase64UUID()).setSource("{}", XContentType.JSON).get();
client().admin().indices().prepareFlush("test").setForce(randomBoolean()).setWaitIfOngoing(true).get();
final FlushStats flushStats = client().admin().indices().prepareStats("test").clear().setFlush(true).get().getTotal().flush;
assertThat(flushStats.getTotal(), greaterThan(flushStats.getPeriodic()));
}
Aggregations