use of org.opensearch.index.shard.DocsStats in project OpenSearch by opensearch-project.
the class TransportRolloverAction method evaluateConditions.
static Map<String, Boolean> evaluateConditions(final Collection<Condition<?>> conditions, @Nullable final DocsStats docsStats, @Nullable final IndexMetadata metadata) {
if (metadata == null) {
return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false));
}
final long numDocs = docsStats == null ? 0 : docsStats.getCount();
final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
final Condition.Stats stats = new Condition.Stats(numDocs, metadata.getCreationDate(), new ByteSizeValue(indexSize));
return conditions.stream().map(condition -> condition.evaluate(stats)).collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched));
}
use of org.opensearch.index.shard.DocsStats in project OpenSearch by opensearch-project.
the class TransportResizeAction method prepareCreateIndexRequest.
// static for unittesting this method
static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ResizeRequest resizeRequest, final ClusterState state, final IntFunction<DocsStats> perShardDocStats, String sourceIndexName, String targetIndexName) {
final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest();
final IndexMetadata metadata = state.metadata().index(sourceIndexName);
if (metadata == null) {
throw new IndexNotFoundException(sourceIndexName);
}
final Settings.Builder targetIndexSettingsBuilder = Settings.builder().put(targetIndex.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
targetIndexSettingsBuilder.remove(IndexMetadata.SETTING_HISTORY_UUID);
final Settings targetIndexSettings = targetIndexSettingsBuilder.build();
final int numShards;
if (IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings);
} else {
assert resizeRequest.getResizeType() != ResizeType.SPLIT : "split must specify the number of shards explicitly";
if (resizeRequest.getResizeType() == ResizeType.SHRINK) {
numShards = 1;
} else {
assert resizeRequest.getResizeType() == ResizeType.CLONE;
numShards = metadata.getNumberOfShards();
}
}
for (int i = 0; i < numShards; i++) {
if (resizeRequest.getResizeType() == ResizeType.SHRINK) {
Set<ShardId> shardIds = IndexMetadata.selectShrinkShards(i, metadata, numShards);
long count = 0;
for (ShardId id : shardIds) {
DocsStats docsStats = perShardDocStats.apply(id.id());
if (docsStats != null) {
count += docsStats.getCount();
}
if (count > IndexWriter.MAX_DOCS) {
throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS + "] docs - too many documents in shards " + shardIds);
}
}
} else if (resizeRequest.getResizeType() == ResizeType.SPLIT) {
Objects.requireNonNull(IndexMetadata.selectSplitShard(i, metadata, numShards));
// we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc.
} else {
Objects.requireNonNull(IndexMetadata.selectCloneShard(i, metadata, numShards));
// we just execute this to ensure we get the right exceptions if the number of shards is wrong etc.
}
}
if (IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) {
throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index");
}
if (IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) {
// if we have a source index with 1 shards it's legal to set this
final boolean splitFromSingleShards = resizeRequest.getResizeType() == ResizeType.SPLIT && metadata.getNumberOfShards() == 1;
if (splitFromSingleShards == false) {
throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize");
}
}
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metadata.getSettings()) && IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(targetIndexSettings) && IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) {
throw new IllegalArgumentException("Can't disable [index.soft_deletes.enabled] setting on resize");
}
String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index";
targetIndex.cause(cause);
Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);
settingsBuilder.put("index.number_of_shards", numShards);
targetIndex.settings(settingsBuilder);
return new CreateIndexClusterStateUpdateRequest(cause, targetIndex.index(), targetIndexName).ackTimeout(targetIndex.timeout()).masterNodeTimeout(targetIndex.masterNodeTimeout()).settings(targetIndex.settings()).aliases(targetIndex.aliases()).waitForActiveShards(targetIndex.waitForActiveShards()).recoverFrom(metadata.getIndex()).resizeType(resizeRequest.getResizeType()).copySettings(resizeRequest.getCopySettings() == null ? false : resizeRequest.getCopySettings());
}
use of org.opensearch.index.shard.DocsStats in project OpenSearch by opensearch-project.
the class CommonStats method add.
public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
docs = new DocsStats();
docs.add(stats.getDocs());
}
} else {
docs.add(stats.getDocs());
}
if (store == null) {
if (stats.getStore() != null) {
store = new StoreStats();
store.add(stats.getStore());
}
} else {
store.add(stats.getStore());
}
if (indexing == null) {
if (stats.getIndexing() != null) {
indexing = new IndexingStats();
indexing.add(stats.getIndexing());
}
} else {
indexing.add(stats.getIndexing());
}
if (get == null) {
if (stats.getGet() != null) {
get = new GetStats();
get.add(stats.getGet());
}
} else {
get.add(stats.getGet());
}
if (search == null) {
if (stats.getSearch() != null) {
search = new SearchStats();
search.add(stats.getSearch());
}
} else {
search.add(stats.getSearch());
}
if (merge == null) {
if (stats.getMerge() != null) {
merge = new MergeStats();
merge.add(stats.getMerge());
}
} else {
merge.add(stats.getMerge());
}
if (refresh == null) {
if (stats.getRefresh() != null) {
refresh = new RefreshStats();
refresh.add(stats.getRefresh());
}
} else {
refresh.add(stats.getRefresh());
}
if (flush == null) {
if (stats.getFlush() != null) {
flush = new FlushStats();
flush.add(stats.getFlush());
}
} else {
flush.add(stats.getFlush());
}
if (warmer == null) {
if (stats.getWarmer() != null) {
warmer = new WarmerStats();
warmer.add(stats.getWarmer());
}
} else {
warmer.add(stats.getWarmer());
}
if (queryCache == null) {
if (stats.getQueryCache() != null) {
queryCache = new QueryCacheStats();
queryCache.add(stats.getQueryCache());
}
} else {
queryCache.add(stats.getQueryCache());
}
if (fieldData == null) {
if (stats.getFieldData() != null) {
fieldData = new FieldDataStats();
fieldData.add(stats.getFieldData());
}
} else {
fieldData.add(stats.getFieldData());
}
if (completion == null) {
if (stats.getCompletion() != null) {
completion = new CompletionStats();
completion.add(stats.getCompletion());
}
} else {
completion.add(stats.getCompletion());
}
if (segments == null) {
if (stats.getSegments() != null) {
segments = new SegmentsStats();
segments.add(stats.getSegments());
}
} else {
segments.add(stats.getSegments());
}
if (translog == null) {
if (stats.getTranslog() != null) {
translog = new TranslogStats();
translog.add(stats.getTranslog());
}
} else {
translog.add(stats.getTranslog());
}
if (requestCache == null) {
if (stats.getRequestCache() != null) {
requestCache = new RequestCacheStats();
requestCache.add(stats.getRequestCache());
}
} else {
requestCache.add(stats.getRequestCache());
}
if (recoveryStats == null) {
if (stats.getRecoveryStats() != null) {
recoveryStats = new RecoveryStats();
recoveryStats.add(stats.getRecoveryStats());
}
} else {
recoveryStats.add(stats.getRecoveryStats());
}
}
use of org.opensearch.index.shard.DocsStats in project OpenSearch by opensearch-project.
the class TransportRolloverActionTests method testEvaluateConditions.
public void testEvaluateConditions() {
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB));
long matchMaxDocs = randomIntBetween(100, 1000);
long notMatchMaxDocs = randomIntBetween(0, 99);
ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB);
final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build();
final IndexMetadata metadata = IndexMetadata.builder(randomAlphaOfLength(10)).creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis()).settings(settings).build();
final Set<Condition<?>> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
Map<String, Boolean> results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), metadata);
assertThat(results.size(), equalTo(3));
for (Boolean matched : results.values()) {
assertThat(matched, equalTo(true));
}
results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, notMatchMaxSize.getBytes()), metadata);
assertThat(results.size(), equalTo(3));
for (Map.Entry<String, Boolean> entry : results.entrySet()) {
if (entry.getKey().equals(maxAgeCondition.toString())) {
assertThat(entry.getValue(), equalTo(true));
} else if (entry.getKey().equals(maxDocsCondition.toString())) {
assertThat(entry.getValue(), equalTo(false));
} else if (entry.getKey().equals(maxSizeCondition.toString())) {
assertThat(entry.getValue(), equalTo(false));
} else {
fail("unknown condition result found " + entry.getKey());
}
}
}
use of org.opensearch.index.shard.DocsStats in project OpenSearch by opensearch-project.
the class RestShardsAction method buildTable.
// package private for testing
Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
Table table = getTableWithHeader(request);
for (ShardRouting shard : state.getState().routingTable().allShards()) {
ShardStats shardStats = stats.asMap().get(shard);
CommonStats commonStats = null;
CommitStats commitStats = null;
if (shardStats != null) {
commonStats = shardStats.getStats();
commitStats = shardStats.getCommitStats();
}
table.startRow();
table.addCell(shard.getIndexName());
table.addCell(shard.id());
if (shard.primary()) {
table.addCell("p");
} else {
table.addCell("r");
}
table.addCell(shard.state());
table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
if (shard.assignedToNode()) {
String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
String nodeId = shard.currentNodeId();
StringBuilder name = new StringBuilder();
name.append(state.getState().nodes().get(shard.currentNodeId()).getName());
if (shard.relocating()) {
String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress();
String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName();
String reloNodeId = shard.relocatingNodeId();
name.append(" -> ");
name.append(reloIp);
name.append(" ");
name.append(reloNodeId);
name.append(" ");
name.append(reloNme);
}
table.addCell(ip);
table.addCell(nodeId);
table.addCell(name);
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
if (shard.unassignedInfo() != null) {
table.addCell(shard.unassignedInfo().getReason());
Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
table.addCell(shard.unassignedInfo().getDetails());
} else {
table.addCell(null);
table.addCell(null);
table.addCell(null);
table.addCell(null);
}
if (shard.recoverySource() != null) {
table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
} else {
table.addCell(null);
}
table.addCell(getOrNull(commonStats, CommonStats::getCompletion, CompletionStats::getSize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getMemorySize));
table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getEvictions));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsCount));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingTime));
table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingCount));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalNumDocs));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalSize));
table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotal));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotalTime));
table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getListeners));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, SearchStats::getOpenContexts));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getIndexWriterMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getVersionMapMemory));
table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getBitsetMemory));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getMaxSeqNo));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getLocalCheckpoint));
table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getGlobalCheckpoint));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::current));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
table.endRow();
}
return table;
}
Aggregations