Search in sources :

Example 1 with FieldDataStats

use of org.opensearch.index.fielddata.FieldDataStats in project OpenSearch by opensearch-project.

the class CommonStats method add.

public void add(CommonStats stats) {
    if (docs == null) {
        if (stats.getDocs() != null) {
            docs = new DocsStats();
            docs.add(stats.getDocs());
        }
    } else {
        docs.add(stats.getDocs());
    }
    if (store == null) {
        if (stats.getStore() != null) {
            store = new StoreStats();
            store.add(stats.getStore());
        }
    } else {
        store.add(stats.getStore());
    }
    if (indexing == null) {
        if (stats.getIndexing() != null) {
            indexing = new IndexingStats();
            indexing.add(stats.getIndexing());
        }
    } else {
        indexing.add(stats.getIndexing());
    }
    if (get == null) {
        if (stats.getGet() != null) {
            get = new GetStats();
            get.add(stats.getGet());
        }
    } else {
        get.add(stats.getGet());
    }
    if (search == null) {
        if (stats.getSearch() != null) {
            search = new SearchStats();
            search.add(stats.getSearch());
        }
    } else {
        search.add(stats.getSearch());
    }
    if (merge == null) {
        if (stats.getMerge() != null) {
            merge = new MergeStats();
            merge.add(stats.getMerge());
        }
    } else {
        merge.add(stats.getMerge());
    }
    if (refresh == null) {
        if (stats.getRefresh() != null) {
            refresh = new RefreshStats();
            refresh.add(stats.getRefresh());
        }
    } else {
        refresh.add(stats.getRefresh());
    }
    if (flush == null) {
        if (stats.getFlush() != null) {
            flush = new FlushStats();
            flush.add(stats.getFlush());
        }
    } else {
        flush.add(stats.getFlush());
    }
    if (warmer == null) {
        if (stats.getWarmer() != null) {
            warmer = new WarmerStats();
            warmer.add(stats.getWarmer());
        }
    } else {
        warmer.add(stats.getWarmer());
    }
    if (queryCache == null) {
        if (stats.getQueryCache() != null) {
            queryCache = new QueryCacheStats();
            queryCache.add(stats.getQueryCache());
        }
    } else {
        queryCache.add(stats.getQueryCache());
    }
    if (fieldData == null) {
        if (stats.getFieldData() != null) {
            fieldData = new FieldDataStats();
            fieldData.add(stats.getFieldData());
        }
    } else {
        fieldData.add(stats.getFieldData());
    }
    if (completion == null) {
        if (stats.getCompletion() != null) {
            completion = new CompletionStats();
            completion.add(stats.getCompletion());
        }
    } else {
        completion.add(stats.getCompletion());
    }
    if (segments == null) {
        if (stats.getSegments() != null) {
            segments = new SegmentsStats();
            segments.add(stats.getSegments());
        }
    } else {
        segments.add(stats.getSegments());
    }
    if (translog == null) {
        if (stats.getTranslog() != null) {
            translog = new TranslogStats();
            translog.add(stats.getTranslog());
        }
    } else {
        translog.add(stats.getTranslog());
    }
    if (requestCache == null) {
        if (stats.getRequestCache() != null) {
            requestCache = new RequestCacheStats();
            requestCache.add(stats.getRequestCache());
        }
    } else {
        requestCache.add(stats.getRequestCache());
    }
    if (recoveryStats == null) {
        if (stats.getRecoveryStats() != null) {
            recoveryStats = new RecoveryStats();
            recoveryStats.add(stats.getRecoveryStats());
        }
    } else {
        recoveryStats.add(stats.getRecoveryStats());
    }
}
Also used : StoreStats(org.opensearch.index.store.StoreStats) RefreshStats(org.opensearch.index.refresh.RefreshStats) IndexingStats(org.opensearch.index.shard.IndexingStats) WarmerStats(org.opensearch.index.warmer.WarmerStats) TranslogStats(org.opensearch.index.translog.TranslogStats) GetStats(org.opensearch.index.get.GetStats) SearchStats(org.opensearch.index.search.stats.SearchStats) RecoveryStats(org.opensearch.index.recovery.RecoveryStats) SegmentsStats(org.opensearch.index.engine.SegmentsStats) FlushStats(org.opensearch.index.flush.FlushStats) QueryCacheStats(org.opensearch.index.cache.query.QueryCacheStats) MergeStats(org.opensearch.index.merge.MergeStats) DocsStats(org.opensearch.index.shard.DocsStats) RequestCacheStats(org.opensearch.index.cache.request.RequestCacheStats) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats)

Example 2 with FieldDataStats

use of org.opensearch.index.fielddata.FieldDataStats in project OpenSearch by opensearch-project.

the class IndexShardTests method testReaderWrapperWorksWithGlobalOrdinals.

public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException {
    CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("_doc", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, wrapper);
    recoverShardFromStore(shard);
    indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fieldType("foo");
    IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(shard.indexSettings.getNodeSettings(), new IndexFieldDataCache.Listener() {
    });
    IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService());
    IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> {
        throw new UnsupportedOperationException("search lookup not available");
    });
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}
Also used : Matchers.hasToString(org.hamcrest.Matchers.hasToString) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) MockFSDirectoryFactory(org.opensearch.test.store.MockFSDirectoryFactory) Arrays(java.util.Arrays) CheckedFunction(org.opensearch.common.CheckedFunction) IndexScopedSettings(org.opensearch.common.settings.IndexScopedSettings) Term(org.apache.lucene.index.Term) Matchers.not(org.hamcrest.Matchers.not) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) Strings(org.opensearch.common.Strings) InternalEngine(org.opensearch.index.engine.InternalEngine) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) IndexFieldDataCache(org.opensearch.index.fielddata.IndexFieldDataCache) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) Path(java.nio.file.Path) NodeEnvironment(org.opensearch.env.NodeEnvironment) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) RegexMatcher.matches(org.opensearch.test.hamcrest.RegexMatcher.matches) Engine(org.opensearch.index.engine.Engine) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) FieldMaskingReader(org.opensearch.test.FieldMaskingReader) FileVisitResult(java.nio.file.FileVisitResult) CountDownLatch(java.util.concurrent.CountDownLatch) VersionType(org.opensearch.index.VersionType) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) Stream(java.util.stream.Stream) Randomness(org.opensearch.common.Randomness) BytesArray(org.opensearch.common.bytes.BytesArray) XContentType(org.opensearch.common.xcontent.XContentType) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) Matchers.in(org.hamcrest.Matchers.in) XContentFactory.jsonBuilder(org.opensearch.common.xcontent.XContentFactory.jsonBuilder) CodecService(org.opensearch.index.codec.CodecService) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) MapperParsingException(org.opensearch.index.mapper.MapperParsingException) ThreadPool(org.opensearch.threadpool.ThreadPool) TestShardRouting.newShardRouting(org.opensearch.cluster.routing.TestShardRouting.newShardRouting) Releasable(org.opensearch.common.lease.Releasable) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) DocIdSeqNoAndSource(org.opensearch.index.engine.DocIdSeqNoAndSource) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) VersionUtils(org.opensearch.test.VersionUtils) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) VersionFieldMapper(org.opensearch.index.mapper.VersionFieldMapper) Matchers.hasSize(org.hamcrest.Matchers.hasSize) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) CorruptionUtils(org.opensearch.test.CorruptionUtils) CommitStats(org.opensearch.index.engine.CommitStats) TopDocs(org.apache.lucene.search.TopDocs) ParseContext(org.opensearch.index.mapper.ParseContext) Versions(org.opensearch.common.lucene.uid.Versions) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Files(java.nio.file.Files) TestTranslog(org.opensearch.index.translog.TestTranslog) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) SourceFieldMapper(org.opensearch.index.mapper.SourceFieldMapper) IndexFieldDataService(org.opensearch.index.fielddata.IndexFieldDataService) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Matchers.sameInstance(org.hamcrest.Matchers.sameInstance) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) Assert(org.junit.Assert) SeqNoFieldMapper(org.opensearch.index.mapper.SeqNoFieldMapper) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) Matchers.either(org.hamcrest.Matchers.either) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) IndexableField(org.apache.lucene.index.IndexableField) Lucene.cleanLuceneIndex(org.opensearch.common.lucene.Lucene.cleanLuceneIndex) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) Releasables(org.opensearch.common.lease.Releasables) CommonStatsFlags(org.opensearch.action.admin.indices.stats.CommonStatsFlags) Matchers.hasKey(org.hamcrest.Matchers.hasKey) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ForceMergeRequest(org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) IndexId(org.opensearch.repositories.IndexId) Matchers.everyItem(org.hamcrest.Matchers.everyItem) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Directory(org.apache.lucene.store.Directory) Assertions(org.opensearch.Assertions) XContentFactory(org.opensearch.common.xcontent.XContentFactory) DummyShardLock(org.opensearch.test.DummyShardLock) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) EngineTestCase(org.opensearch.index.engine.EngineTestCase) CyclicBarrier(java.util.concurrent.CyclicBarrier) DeleteResult(org.opensearch.index.engine.Engine.DeleteResult) BytesRef(org.apache.lucene.util.BytesRef) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) SnapshotId(org.opensearch.snapshots.SnapshotId) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) DirectoryReader(org.apache.lucene.index.DirectoryReader) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) List(java.util.List) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexSettings(org.opensearch.index.IndexSettings) ShardRoutingHelper(org.opensearch.cluster.routing.ShardRoutingHelper) Uid(org.opensearch.index.mapper.Uid) TranslogStats(org.opensearch.index.translog.TranslogStats) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IntStream(java.util.stream.IntStream) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EngineConfig(org.opensearch.index.engine.EngineConfig) StoreUtils(org.opensearch.index.store.StoreUtils) IndicesFieldDataCache(org.opensearch.indices.fielddata.cache.IndicesFieldDataCache) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexFieldData(org.opensearch.index.fielddata.IndexFieldData) SourceToParse(org.opensearch.index.mapper.SourceToParse) Charset(java.nio.charset.Charset) Translog(org.opensearch.index.translog.Translog) UUIDs(org.opensearch.common.UUIDs) IndicesQueryCache(org.opensearch.indices.IndicesQueryCache) StoreStats(org.opensearch.index.store.StoreStats) RetentionLease(org.opensearch.index.seqno.RetentionLease) StreamInput(org.opensearch.common.io.stream.StreamInput) InternalEngineFactory(org.opensearch.index.engine.InternalEngineFactory) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.oneOf(org.hamcrest.Matchers.oneOf) RecoveryTarget(org.opensearch.indices.recovery.RecoveryTarget) LongFunction(java.util.function.LongFunction) Collections.emptySet(java.util.Collections.emptySet) AllocationId(org.opensearch.cluster.routing.AllocationId) Semaphore(java.util.concurrent.Semaphore) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) EMPTY_PARAMS(org.opensearch.common.xcontent.ToXContent.EMPTY_PARAMS) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) TermQuery(org.apache.lucene.search.TermQuery) Constants(org.apache.lucene.util.Constants) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) FilterDirectory(org.apache.lucene.store.FilterDirectory) Snapshot(org.opensearch.snapshots.Snapshot) IndexRequest(org.opensearch.action.index.IndexRequest) Collections(java.util.Collections) DirectoryReader(org.apache.lucene.index.DirectoryReader) FieldMaskingReader(org.opensearch.test.FieldMaskingReader) IOException(java.io.IOException) IndexFieldDataCache(org.opensearch.index.fielddata.IndexFieldDataCache) IndicesFieldDataCache(org.opensearch.indices.fielddata.cache.IndicesFieldDataCache) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) IndexFieldDataService(org.opensearch.index.fielddata.IndexFieldDataService) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) IndexFieldData(org.opensearch.index.fielddata.IndexFieldData) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) IndexScopedSettings(org.opensearch.common.settings.IndexScopedSettings) Settings(org.opensearch.common.settings.Settings) IndexSettings(org.opensearch.index.IndexSettings) InternalEngine(org.opensearch.index.engine.InternalEngine) Engine(org.opensearch.index.engine.Engine) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats)

Example 3 with FieldDataStats

use of org.opensearch.index.fielddata.FieldDataStats in project OpenSearch by opensearch-project.

the class RestShardsAction method buildTable.

// package private for testing
Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
    Table table = getTableWithHeader(request);
    for (ShardRouting shard : state.getState().routingTable().allShards()) {
        ShardStats shardStats = stats.asMap().get(shard);
        CommonStats commonStats = null;
        CommitStats commitStats = null;
        if (shardStats != null) {
            commonStats = shardStats.getStats();
            commitStats = shardStats.getCommitStats();
        }
        table.startRow();
        table.addCell(shard.getIndexName());
        table.addCell(shard.id());
        if (shard.primary()) {
            table.addCell("p");
        } else {
            table.addCell("r");
        }
        table.addCell(shard.state());
        table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
        if (shard.assignedToNode()) {
            String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
            String nodeId = shard.currentNodeId();
            StringBuilder name = new StringBuilder();
            name.append(state.getState().nodes().get(shard.currentNodeId()).getName());
            if (shard.relocating()) {
                String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress();
                String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName();
                String reloNodeId = shard.relocatingNodeId();
                name.append(" -> ");
                name.append(reloIp);
                name.append(" ");
                name.append(reloNodeId);
                name.append(" ");
                name.append(reloNme);
            }
            table.addCell(ip);
            table.addCell(nodeId);
            table.addCell(name);
        } else {
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
        }
        table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
        if (shard.unassignedInfo() != null) {
            table.addCell(shard.unassignedInfo().getReason());
            Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
            table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
            table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
            table.addCell(shard.unassignedInfo().getDetails());
        } else {
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
        }
        if (shard.recoverySource() != null) {
            table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
        } else {
            table.addCell(null);
        }
        table.addCell(getOrNull(commonStats, CommonStats::getCompletion, CompletionStats::getSize));
        table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getMemorySize));
        table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getEvictions));
        table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getMemorySize));
        table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getEvictions));
        table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::current));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsCount));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingCount));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentSize));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalNumDocs));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalSize));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getListeners));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, SearchStats::getOpenContexts));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getIndexWriterMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getVersionMapMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getBitsetMemory));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getMaxSeqNo));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getLocalCheckpoint));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getGlobalCheckpoint));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::current));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
        table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
        table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
        table.endRow();
    }
    return table;
}
Also used : ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) Collections.unmodifiableList(java.util.Collections.unmodifiableList) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) ClusterStateResponse(org.opensearch.action.admin.cluster.state.ClusterStateResponse) SearchStats(org.opensearch.index.search.stats.SearchStats) Table(org.opensearch.common.Table) Function(java.util.function.Function) Strings(org.opensearch.common.Strings) MergeStats(org.opensearch.index.merge.MergeStats) RestActionListener(org.opensearch.rest.action.RestActionListener) RefreshStats(org.opensearch.index.refresh.RefreshStats) Locale(java.util.Locale) Arrays.asList(java.util.Arrays.asList) SegmentsStats(org.opensearch.index.engine.SegmentsStats) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) StoreStats(org.opensearch.index.store.StoreStats) CommitStats(org.opensearch.index.engine.CommitStats) RestResponseListener(org.opensearch.rest.action.RestResponseListener) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats) TimeValue(org.opensearch.common.unit.TimeValue) NodeClient(org.opensearch.client.node.NodeClient) GET(org.opensearch.rest.RestRequest.Method.GET) RestRequest(org.opensearch.rest.RestRequest) DocsStats(org.opensearch.index.shard.DocsStats) FlushStats(org.opensearch.index.flush.FlushStats) IndicesStatsRequest(org.opensearch.action.admin.indices.stats.IndicesStatsRequest) Instant(java.time.Instant) GetStats(org.opensearch.index.get.GetStats) RestResponse(org.opensearch.rest.RestResponse) WarmerStats(org.opensearch.index.warmer.WarmerStats) ShardRouting(org.opensearch.cluster.routing.ShardRouting) QueryCacheStats(org.opensearch.index.cache.query.QueryCacheStats) Engine(org.opensearch.index.engine.Engine) ClusterStateRequest(org.opensearch.action.admin.cluster.state.ClusterStateRequest) List(java.util.List) IndicesStatsResponse(org.opensearch.action.admin.indices.stats.IndicesStatsResponse) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) Table(org.opensearch.common.Table) CommitStats(org.opensearch.index.engine.CommitStats) Instant(java.time.Instant) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 4 with FieldDataStats

use of org.opensearch.index.fielddata.FieldDataStats in project OpenSearch by opensearch-project.

the class TransportRolloverActionTests method randomIndicesStatsResponse.

public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadata[] indices) {
    List<ShardStats> shardStats = new ArrayList<>();
    for (final IndexMetadata index : indices) {
        int numShards = randomIntBetween(1, 3);
        // -1 means there is no primary shard.
        int primaryIdx = randomIntBetween(-1, numShards - 1);
        for (int i = 0; i < numShards; i++) {
            ShardId shardId = new ShardId(index.getIndex(), i);
            boolean primary = (i == primaryIdx);
            Path path = createTempDir().resolve("indices").resolve(index.getIndexUUID()).resolve(String.valueOf(i));
            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary, primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
            shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
            shardRouting = shardRouting.moveToStarted();
            CommonStats stats = new CommonStats();
            stats.fieldData = new FieldDataStats();
            stats.queryCache = new QueryCacheStats();
            stats.docs = new DocsStats();
            stats.store = new StoreStats();
            stats.indexing = new IndexingStats();
            stats.search = new SearchStats();
            stats.segments = new SegmentsStats();
            stats.merge = new MergeStats();
            stats.refresh = new RefreshStats();
            stats.completion = new CompletionStats();
            stats.requestCache = new RequestCacheStats();
            stats.get = new GetStats();
            stats.flush = new FlushStats();
            stats.warmer = new WarmerStats();
            shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null));
        }
    }
    return IndicesStatsTests.newIndicesStatsResponse(shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList());
}
Also used : StoreStats(org.opensearch.index.store.StoreStats) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) RefreshStats(org.opensearch.index.refresh.RefreshStats) ArrayList(java.util.ArrayList) GetStats(org.opensearch.index.get.GetStats) SegmentsStats(org.opensearch.index.engine.SegmentsStats) ShardId(org.opensearch.index.shard.ShardId) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) FlushStats(org.opensearch.index.flush.FlushStats) ShardPath(org.opensearch.index.shard.ShardPath) QueryCacheStats(org.opensearch.index.cache.query.QueryCacheStats) DocsStats(org.opensearch.index.shard.DocsStats) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) Path(java.nio.file.Path) ShardPath(org.opensearch.index.shard.ShardPath) IndexingStats(org.opensearch.index.shard.IndexingStats) WarmerStats(org.opensearch.index.warmer.WarmerStats) SearchStats(org.opensearch.index.search.stats.SearchStats) MergeStats(org.opensearch.index.merge.MergeStats) ShardRouting(org.opensearch.cluster.routing.ShardRouting) RequestCacheStats(org.opensearch.index.cache.request.RequestCacheStats) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats)

Example 5 with FieldDataStats

use of org.opensearch.index.fielddata.FieldDataStats in project OpenSearch by opensearch-project.

the class RestNodesAction method buildTable.

Table buildTable(boolean fullId, RestRequest req, ClusterStateResponse state, NodesInfoResponse nodesInfo, NodesStatsResponse nodesStats) {
    DiscoveryNodes nodes = state.getState().nodes();
    String masterId = nodes.getMasterNodeId();
    Table table = getTableWithHeader(req);
    for (DiscoveryNode node : nodes) {
        NodeInfo info = nodesInfo.getNodesMap().get(node.getId());
        NodeStats stats = nodesStats.getNodesMap().get(node.getId());
        JvmInfo jvmInfo = info == null ? null : info.getInfo(JvmInfo.class);
        JvmStats jvmStats = stats == null ? null : stats.getJvm();
        FsInfo fsInfo = stats == null ? null : stats.getFs();
        OsStats osStats = stats == null ? null : stats.getOs();
        ProcessStats processStats = stats == null ? null : stats.getProcess();
        NodeIndicesStats indicesStats = stats == null ? null : stats.getIndices();
        table.startRow();
        table.addCell(fullId ? node.getId() : Strings.substring(node.getId(), 0, 4));
        table.addCell(info == null ? null : info.getInfo(ProcessInfo.class).getId());
        table.addCell(node.getHostAddress());
        table.addCell(node.getAddress().address().getPort());
        final HttpInfo httpInfo = info == null ? null : info.getInfo(HttpInfo.class);
        if (httpInfo != null) {
            TransportAddress transportAddress = httpInfo.getAddress().publishAddress();
            table.addCell(NetworkAddress.format(transportAddress.address()));
        } else {
            table.addCell("-");
        }
        table.addCell(node.getVersion().toString());
        table.addCell(info == null ? null : info.getBuild().type().displayName());
        table.addCell(info == null ? null : info.getBuild().hash());
        table.addCell(jvmInfo == null ? null : jvmInfo.version());
        ByteSizeValue diskTotal = null;
        ByteSizeValue diskUsed = null;
        ByteSizeValue diskAvailable = null;
        String diskUsedPercent = null;
        if (fsInfo != null) {
            diskTotal = fsInfo.getTotal().getTotal();
            diskAvailable = fsInfo.getTotal().getAvailable();
            diskUsed = new ByteSizeValue(diskTotal.getBytes() - diskAvailable.getBytes());
            double diskUsedRatio = diskTotal.getBytes() == 0 ? 1.0 : (double) diskUsed.getBytes() / diskTotal.getBytes();
            diskUsedPercent = String.format(Locale.ROOT, "%.2f", 100.0 * diskUsedRatio);
        }
        table.addCell(diskTotal);
        table.addCell(diskUsed);
        table.addCell(diskAvailable);
        table.addCell(diskUsedPercent);
        table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsed());
        table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsedPercent());
        table.addCell(jvmInfo == null ? null : jvmInfo.getMem().getHeapMax());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getUsed());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getUsedPercent());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getTotal());
        table.addCell(processStats == null ? null : processStats.getOpenFileDescriptors());
        table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors()));
        table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors());
        table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent()));
        boolean hasLoadAverage = osStats != null && osStats.getCpu().getLoadAverage() != null;
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[0] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[0]));
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[1] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[1]));
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[2] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[2]));
        table.addCell(jvmStats == null ? null : jvmStats.getUptime());
        final String roles;
        if (node.getRoles().isEmpty()) {
            roles = "-";
        } else {
            roles = node.getRoles().stream().map(DiscoveryNodeRole::roleNameAbbreviation).sorted().collect(Collectors.joining());
        }
        table.addCell(roles);
        table.addCell(masterId == null ? "x" : masterId.equals(node.getId()) ? "*" : "-");
        table.addCell(node.getName());
        CompletionStats completionStats = indicesStats == null ? null : stats.getIndices().getCompletion();
        table.addCell(completionStats == null ? null : completionStats.getSize());
        FieldDataStats fdStats = indicesStats == null ? null : stats.getIndices().getFieldData();
        table.addCell(fdStats == null ? null : fdStats.getMemorySize());
        table.addCell(fdStats == null ? null : fdStats.getEvictions());
        QueryCacheStats fcStats = indicesStats == null ? null : indicesStats.getQueryCache();
        table.addCell(fcStats == null ? null : fcStats.getMemorySize());
        table.addCell(fcStats == null ? null : fcStats.getEvictions());
        table.addCell(fcStats == null ? null : fcStats.getHitCount());
        table.addCell(fcStats == null ? null : fcStats.getMissCount());
        RequestCacheStats qcStats = indicesStats == null ? null : indicesStats.getRequestCache();
        table.addCell(qcStats == null ? null : qcStats.getMemorySize());
        table.addCell(qcStats == null ? null : qcStats.getEvictions());
        table.addCell(qcStats == null ? null : qcStats.getHitCount());
        table.addCell(qcStats == null ? null : qcStats.getMissCount());
        FlushStats flushStats = indicesStats == null ? null : indicesStats.getFlush();
        table.addCell(flushStats == null ? null : flushStats.getTotal());
        table.addCell(flushStats == null ? null : flushStats.getTotalTime());
        GetStats getStats = indicesStats == null ? null : indicesStats.getGet();
        table.addCell(getStats == null ? null : getStats.current());
        table.addCell(getStats == null ? null : getStats.getTime());
        table.addCell(getStats == null ? null : getStats.getCount());
        table.addCell(getStats == null ? null : getStats.getExistsTime());
        table.addCell(getStats == null ? null : getStats.getExistsCount());
        table.addCell(getStats == null ? null : getStats.getMissingTime());
        table.addCell(getStats == null ? null : getStats.getMissingCount());
        IndexingStats indexingStats = indicesStats == null ? null : indicesStats.getIndexing();
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteCurrent());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteTime());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteCount());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCurrent());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount());
        MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge();
        table.addCell(mergeStats == null ? null : mergeStats.getCurrent());
        table.addCell(mergeStats == null ? null : mergeStats.getCurrentNumDocs());
        table.addCell(mergeStats == null ? null : mergeStats.getCurrentSize());
        table.addCell(mergeStats == null ? null : mergeStats.getTotal());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalNumDocs());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalSize());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalTime());
        RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
        table.addCell(refreshStats == null ? null : refreshStats.getTotal());
        table.addCell(refreshStats == null ? null : refreshStats.getTotalTime());
        table.addCell(refreshStats == null ? null : refreshStats.getExternalTotal());
        table.addCell(refreshStats == null ? null : refreshStats.getExternalTotalTime());
        table.addCell(refreshStats == null ? null : refreshStats.getListeners());
        ScriptStats scriptStats = stats == null ? null : stats.getScriptStats();
        table.addCell(scriptStats == null ? null : scriptStats.getCompilations());
        table.addCell(scriptStats == null ? null : scriptStats.getCacheEvictions());
        table.addCell(scriptStats == null ? null : scriptStats.getCompilationLimitTriggered());
        SearchStats searchStats = indicesStats == null ? null : indicesStats.getSearch();
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchCount());
        table.addCell(searchStats == null ? null : searchStats.getOpenContexts());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCount());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCount());
        SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
        table.addCell(segmentsStats == null ? null : segmentsStats.getCount());
        table.addCell(segmentsStats == null ? null : segmentsStats.getZeroMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getIndexWriterMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCount());
        table.endRow();
    }
    return table;
}
Also used : DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) JvmInfo(org.opensearch.monitor.jvm.JvmInfo) RefreshStats(org.opensearch.index.refresh.RefreshStats) TransportAddress(org.opensearch.common.transport.TransportAddress) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) GetStats(org.opensearch.index.get.GetStats) SegmentsStats(org.opensearch.index.engine.SegmentsStats) HttpInfo(org.opensearch.http.HttpInfo) NodeStats(org.opensearch.action.admin.cluster.node.stats.NodeStats) FsInfo(org.opensearch.monitor.fs.FsInfo) FlushStats(org.opensearch.index.flush.FlushStats) QueryCacheStats(org.opensearch.index.cache.query.QueryCacheStats) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) JvmStats(org.opensearch.monitor.jvm.JvmStats) ScriptStats(org.opensearch.script.ScriptStats) ProcessStats(org.opensearch.monitor.process.ProcessStats) Table(org.opensearch.common.Table) IndexingStats(org.opensearch.index.shard.IndexingStats) OsStats(org.opensearch.monitor.os.OsStats) ProcessInfo(org.opensearch.monitor.process.ProcessInfo) SearchStats(org.opensearch.index.search.stats.SearchStats) NodeInfo(org.opensearch.action.admin.cluster.node.info.NodeInfo) MergeStats(org.opensearch.index.merge.MergeStats) NodeIndicesStats(org.opensearch.indices.NodeIndicesStats) RequestCacheStats(org.opensearch.index.cache.request.RequestCacheStats) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats)

Aggregations

FieldDataStats (org.opensearch.index.fielddata.FieldDataStats)5 QueryCacheStats (org.opensearch.index.cache.query.QueryCacheStats)4 SegmentsStats (org.opensearch.index.engine.SegmentsStats)4 FlushStats (org.opensearch.index.flush.FlushStats)4 GetStats (org.opensearch.index.get.GetStats)4 MergeStats (org.opensearch.index.merge.MergeStats)4 RefreshStats (org.opensearch.index.refresh.RefreshStats)4 SearchStats (org.opensearch.index.search.stats.SearchStats)4 StoreStats (org.opensearch.index.store.StoreStats)4 CompletionStats (org.opensearch.search.suggest.completion.CompletionStats)4 CommonStats (org.opensearch.action.admin.indices.stats.CommonStats)3 ShardStats (org.opensearch.action.admin.indices.stats.ShardStats)3 ShardRouting (org.opensearch.cluster.routing.ShardRouting)3 UnassignedInfo (org.opensearch.cluster.routing.UnassignedInfo)3 DocsStats (org.opensearch.index.shard.DocsStats)3 WarmerStats (org.opensearch.index.warmer.WarmerStats)3 Path (java.nio.file.Path)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Locale (java.util.Locale)2