Examples with CommitStats org.opensearch.index.engine.CommitStats used on opensource projects

Example 1 with CommitStats

use of org.opensearch.index.engine.CommitStats in project OpenSearch by opensearch-project.

the class IndexShardIT method testMaybeFlush.

public void testMaybeFlush() throws Exception {
    createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
    ensureGreen();
    IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    IndexService test = indicesService.indexService(resolveIndex("test"));
    IndexShard shard = test.getShardOrNull(0);
    assertFalse(shard.shouldPeriodicallyFlush());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(135, /* size of the operation + one generation header&footer*/
    ByteSizeUnit.BYTES)).build()).get();
    client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertFalse(shard.shouldPeriodicallyFlush());
    shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse("test", "_doc", "1", new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
    assertTrue(shard.shouldPeriodicallyFlush());
    final Translog translog = getTranslog(shard);
    assertEquals(2, translog.stats().getUncommittedOperations());
    assertThat(shard.flushStats().getTotal(), equalTo(0L));
    client().prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertThat(shard.getLastKnownGlobalCheckpoint(), equalTo(2L));
    assertBusy(() -> {
        // this is async
        assertFalse(shard.shouldPeriodicallyFlush());
        assertThat(shard.flushStats().getPeriodic(), equalTo(1L));
        assertThat(shard.flushStats().getTotal(), equalTo(1L));
    });
    shard.sync();
    assertThat(shard.getLastSyncedGlobalCheckpoint(), equalTo(2L));
    assertThat("last commit [" + shard.commitStats().getUserData() + "]", translog.stats().getUncommittedOperations(), equalTo(0));
    long size = Math.max(translog.stats().getUncommittedSizeInBytes(), Translog.DEFAULT_HEADER_SIZE_IN_BYTES + 1);
    logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)).build()).get();
    client().prepareDelete("test", "2").get();
    logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    assertBusy(() -> {
        // this is async
        final TranslogStats translogStats = translog.stats();
        final CommitStats commitStats = shard.commitStats();
        final FlushStats flushStats = shard.flushStats();
        logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal());
        assertFalse(shard.shouldPeriodicallyFlush());
    });
    shard.sync();
    assertEquals(0, translog.stats().getUncommittedOperations());
}

Example 2 with CommitStats

use of org.opensearch.index.engine.CommitStats in project OpenSearch by opensearch-project.

the class TransportClusterStatsAction method nodeOperation.

@Override
protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
    NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false);
    NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, true, false, false, false, false);
    List<ShardStats> shardsStats = new ArrayList<>();
    for (IndexService indexService : indicesService) {
        for (IndexShard indexShard : indexService) {
            if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                // only report on fully started shards
                CommitStats commitStats;
                SeqNoStats seqNoStats;
                RetentionLeaseStats retentionLeaseStats;
                try {
                    commitStats = indexShard.commitStats();
                    seqNoStats = indexShard.seqNoStats();
                    retentionLeaseStats = indexShard.getRetentionLeaseStats();
                } catch (final AlreadyClosedException e) {
                    // shard is closed - no stats is fine
                    commitStats = null;
                    seqNoStats = null;
                    retentionLeaseStats = null;
                }
                shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), commitStats, seqNoStats, retentionLeaseStats));
            }
        }
    }
    ClusterHealthStatus clusterStatus = null;
    if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
        clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus();
    }
    return new ClusterStatsNodeResponse(nodeInfo.getNode(), clusterStatus, nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[shardsStats.size()]));
}

Example 3 with CommitStats

use of org.opensearch.index.engine.CommitStats in project OpenSearch by opensearch-project.

the class IndexShardTests method testOnCloseStats.

public void testOnCloseStats() throws IOException {
    final IndexShard indexShard = newStartedShard(true);
    for (int i = 0; i < 3; i++) {
        indexDoc(indexShard, "_doc", "" + i, "{\"foo\" : \"" + randomAlphaOfLength(10) + "\"}");
        // produce segments
        indexShard.refresh("test");
    }
    // check stats on closed and on opened shard
    if (randomBoolean()) {
        closeShards(indexShard);
        expectThrows(AlreadyClosedException.class, () -> indexShard.seqNoStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.commitStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.storeStats());
    } else {
        final SeqNoStats seqNoStats = indexShard.seqNoStats();
        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(2L));
        final CommitStats commitStats = indexShard.commitStats();
        assertThat(commitStats.getGeneration(), equalTo(2L));
        final StoreStats storeStats = indexShard.storeStats();
        assertThat(storeStats.sizeInBytes(), greaterThan(0L));
        closeShards(indexShard);
    }
}

Example 4 with CommitStats

use of org.opensearch.index.engine.CommitStats in project OpenSearch by opensearch-project.

the class IndicesService method indexShardStats.

IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) {
    if (indexShard.routingEntry() == null) {
        return null;
    }
    CommitStats commitStats;
    SeqNoStats seqNoStats;
    RetentionLeaseStats retentionLeaseStats;
    try {
        commitStats = indexShard.commitStats();
        seqNoStats = indexShard.seqNoStats();
        retentionLeaseStats = indexShard.getRetentionLeaseStats();
    } catch (AlreadyClosedException e) {
        // shard is closed - no stats is fine
        commitStats = null;
        seqNoStats = null;
        retentionLeaseStats = null;
    }
    return new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), commitStats, seqNoStats, retentionLeaseStats) });
}

Example 5 with CommitStats

use of org.opensearch.index.engine.CommitStats in project OpenSearch by opensearch-project.

the class IndicesStatsTests method testCommitStats.

public void testCommitStats() throws Exception {
    createIndex("test");
    ensureGreen("test");
    IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get();
    for (ShardStats shardStats : rsp.getIndex("test").getShards()) {
        final CommitStats commitStats = shardStats.getCommitStats();
        assertNotNull(commitStats);
        assertThat(commitStats.getGeneration(), greaterThan(0L));
        assertThat(commitStats.getId(), notNullValue());
        assertThat(commitStats.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY));
    }
}