Examples with StoreStats org.opensearch.index.store.StoreStats used on opensource projects

Example 1 with StoreStats

use of org.opensearch.index.store.StoreStats in project OpenSearch by opensearch-project.

the class InternalClusterInfoService method buildShardLevelInfo.

static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder<String, Long> shardSizes, ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath, Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace.Builder> reservedSpaceByShard) {
    for (ShardStats s : stats) {
        final ShardRouting shardRouting = s.getShardRouting();
        newShardRoutingToDataPath.put(shardRouting, s.getDataPath());
        final StoreStats storeStats = s.getStats().getStore();
        if (storeStats == null) {
            continue;
        }
        final long size = storeStats.sizeInBytes();
        final long reserved = storeStats.getReservedSize().getBytes();
        final String shardIdentifier = ClusterInfo.shardIdentifierFromRouting(shardRouting);
        logger.trace("shard: {} size: {} reserved: {}", shardIdentifier, size, reserved);
        shardSizes.put(shardIdentifier, size);
        if (reserved != StoreStats.UNKNOWN_RESERVED_BYTES) {
            final ClusterInfo.ReservedSpace.Builder reservedSpaceBuilder = reservedSpaceByShard.computeIfAbsent(new ClusterInfo.NodeAndPath(shardRouting.currentNodeId(), s.getDataPath()), t -> new ClusterInfo.ReservedSpace.Builder());
            reservedSpaceBuilder.add(shardRouting.shardId(), reserved);
        }
    }
}

Example 2 with StoreStats

use of org.opensearch.index.store.StoreStats in project OpenSearch by opensearch-project.

the class CommonStats method add.

public void add(CommonStats stats) {
    if (docs == null) {
        if (stats.getDocs() != null) {
            docs = new DocsStats();
            docs.add(stats.getDocs());
        }
    } else {
        docs.add(stats.getDocs());
    }
    if (store == null) {
        if (stats.getStore() != null) {
            store = new StoreStats();
            store.add(stats.getStore());
        }
    } else {
        store.add(stats.getStore());
    }
    if (indexing == null) {
        if (stats.getIndexing() != null) {
            indexing = new IndexingStats();
            indexing.add(stats.getIndexing());
        }
    } else {
        indexing.add(stats.getIndexing());
    }
    if (get == null) {
        if (stats.getGet() != null) {
            get = new GetStats();
            get.add(stats.getGet());
        }
    } else {
        get.add(stats.getGet());
    }
    if (search == null) {
        if (stats.getSearch() != null) {
            search = new SearchStats();
            search.add(stats.getSearch());
        }
    } else {
        search.add(stats.getSearch());
    }
    if (merge == null) {
        if (stats.getMerge() != null) {
            merge = new MergeStats();
            merge.add(stats.getMerge());
        }
    } else {
        merge.add(stats.getMerge());
    }
    if (refresh == null) {
        if (stats.getRefresh() != null) {
            refresh = new RefreshStats();
            refresh.add(stats.getRefresh());
        }
    } else {
        refresh.add(stats.getRefresh());
    }
    if (flush == null) {
        if (stats.getFlush() != null) {
            flush = new FlushStats();
            flush.add(stats.getFlush());
        }
    } else {
        flush.add(stats.getFlush());
    }
    if (warmer == null) {
        if (stats.getWarmer() != null) {
            warmer = new WarmerStats();
            warmer.add(stats.getWarmer());
        }
    } else {
        warmer.add(stats.getWarmer());
    }
    if (queryCache == null) {
        if (stats.getQueryCache() != null) {
            queryCache = new QueryCacheStats();
            queryCache.add(stats.getQueryCache());
        }
    } else {
        queryCache.add(stats.getQueryCache());
    }
    if (fieldData == null) {
        if (stats.getFieldData() != null) {
            fieldData = new FieldDataStats();
            fieldData.add(stats.getFieldData());
        }
    } else {
        fieldData.add(stats.getFieldData());
    }
    if (completion == null) {
        if (stats.getCompletion() != null) {
            completion = new CompletionStats();
            completion.add(stats.getCompletion());
        }
    } else {
        completion.add(stats.getCompletion());
    }
    if (segments == null) {
        if (stats.getSegments() != null) {
            segments = new SegmentsStats();
            segments.add(stats.getSegments());
        }
    } else {
        segments.add(stats.getSegments());
    }
    if (translog == null) {
        if (stats.getTranslog() != null) {
            translog = new TranslogStats();
            translog.add(stats.getTranslog());
        }
    } else {
        translog.add(stats.getTranslog());
    }
    if (requestCache == null) {
        if (stats.getRequestCache() != null) {
            requestCache = new RequestCacheStats();
            requestCache.add(stats.getRequestCache());
        }
    } else {
        requestCache.add(stats.getRequestCache());
    }
    if (recoveryStats == null) {
        if (stats.getRecoveryStats() != null) {
            recoveryStats = new RecoveryStats();
            recoveryStats.add(stats.getRecoveryStats());
        }
    } else {
        recoveryStats.add(stats.getRecoveryStats());
    }
}

Example 3 with StoreStats

use of org.opensearch.index.store.StoreStats in project OpenSearch by opensearch-project.

the class IndexShardTests method testOnCloseStats.

public void testOnCloseStats() throws IOException {
    final IndexShard indexShard = newStartedShard(true);
    for (int i = 0; i < 3; i++) {
        indexDoc(indexShard, "_doc", "" + i, "{\"foo\" : \"" + randomAlphaOfLength(10) + "\"}");
        // produce segments
        indexShard.refresh("test");
    }
    // check stats on closed and on opened shard
    if (randomBoolean()) {
        closeShards(indexShard);
        expectThrows(AlreadyClosedException.class, () -> indexShard.seqNoStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.commitStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.storeStats());
    } else {
        final SeqNoStats seqNoStats = indexShard.seqNoStats();
        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(2L));
        final CommitStats commitStats = indexShard.commitStats();
        assertThat(commitStats.getGeneration(), equalTo(2L));
        final StoreStats storeStats = indexShard.storeStats();
        assertThat(storeStats.sizeInBytes(), greaterThan(0L));
        closeShards(indexShard);
    }
}

Example 4 with StoreStats

use of org.opensearch.index.store.StoreStats in project OpenSearch by opensearch-project.

the class DiskUsageTests method testFillShardLevelInfo.

public void testFillShardLevelInfo() {
    final Index index = new Index("test", "0xdeadbeef");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, "node1");
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
    CommonStats commonStats0 = new CommonStats();
    commonStats0.store = new StoreStats(100, 0L);
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, "node2");
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
    CommonStats commonStats1 = new CommonStats();
    commonStats1.store = new StoreStats(1000, 0L);
    ShardStats[] stats = new ShardStats[] { new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0, null, null, null), new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1, null, null, null) };
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
    ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).build();
    InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath, new HashMap<>());
    assertEquals(2, shardSizes.size());
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_0)));
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_1)));
    assertEquals(100L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_0)).longValue());
    assertEquals(1000L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_1)).longValue());
    assertEquals(2, routingToPath.size());
    assertTrue(routingToPath.containsKey(test_0));
    assertTrue(routingToPath.containsKey(test_1));
    assertEquals(test0Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_0));
    assertEquals(test1Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_1));
}

Example 5 with StoreStats

use of org.opensearch.index.store.StoreStats in project OpenSearch by opensearch-project.

the class RestShardsAction method buildTable.

// package private for testing
Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) {
    Table table = getTableWithHeader(request);
    for (ShardRouting shard : state.getState().routingTable().allShards()) {
        ShardStats shardStats = stats.asMap().get(shard);
        CommonStats commonStats = null;
        CommitStats commitStats = null;
        if (shardStats != null) {
            commonStats = shardStats.getStats();
            commitStats = shardStats.getCommitStats();
        }
        table.startRow();
        table.addCell(shard.getIndexName());
        table.addCell(shard.id());
        if (shard.primary()) {
            table.addCell("p");
        } else {
            table.addCell("r");
        }
        table.addCell(shard.state());
        table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize));
        if (shard.assignedToNode()) {
            String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress();
            String nodeId = shard.currentNodeId();
            StringBuilder name = new StringBuilder();
            name.append(state.getState().nodes().get(shard.currentNodeId()).getName());
            if (shard.relocating()) {
                String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress();
                String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName();
                String reloNodeId = shard.relocatingNodeId();
                name.append(" -> ");
                name.append(reloIp);
                name.append(" ");
                name.append(reloNodeId);
                name.append(" ");
                name.append(reloNme);
            }
            table.addCell(ip);
            table.addCell(nodeId);
            table.addCell(name);
        } else {
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
        }
        table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
        if (shard.unassignedInfo() != null) {
            table.addCell(shard.unassignedInfo().getReason());
            Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
            table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
            table.addCell(TimeValue.timeValueMillis(System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()));
            table.addCell(shard.unassignedInfo().getDetails());
        } else {
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
            table.addCell(null);
        }
        if (shard.recoverySource() != null) {
            table.addCell(shard.recoverySource().getType().toString().toLowerCase(Locale.ROOT));
        } else {
            table.addCell(null);
        }
        table.addCell(getOrNull(commonStats, CommonStats::getCompletion, CompletionStats::getSize));
        table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getMemorySize));
        table.addCell(getOrNull(commonStats, CommonStats::getFieldData, FieldDataStats::getEvictions));
        table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getMemorySize));
        table.addCell(getOrNull(commonStats, CommonStats::getQueryCache, QueryCacheStats::getEvictions));
        table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getFlush, FlushStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::current));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getExistsCount));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingTime));
        table.addCell(getOrNull(commonStats, CommonStats::getGet, GetStats::getMissingCount));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getDeleteCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentSize));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalNumDocs));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalSize));
        table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotal));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getExternalTotalTime));
        table.addCell(getOrNull(commonStats, CommonStats::getRefresh, RefreshStats::getListeners));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getFetchCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, SearchStats::getOpenContexts));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
        table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getIndexWriterMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getVersionMapMemory));
        table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getBitsetMemory));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getMaxSeqNo));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getLocalCheckpoint));
        table.addCell(getOrNull(shardStats, ShardStats::getSeqNoStats, SeqNoStats::getGlobalCheckpoint));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::current));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total));
        table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime));
        table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
        table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
        table.endRow();
    }
    return table;
}