Examples with SnapshotStats org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats used on opensource projects

Example 1 with SnapshotStats

use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats in project OpenSearch by opensearch-project.

the class BlobStoreIncrementalityIT method assertTwoIdenticalShardSnapshots.

private void assertTwoIdenticalShardSnapshots(String repo, String indexName, String snapshot1, String snapshot2) {
    logger.info("--> asserting that snapshots [{}] and [{}] are referring to the same files in the repository", snapshot1, snapshot2);
    final SnapshotStats firstSnapshotShardStatus = getStats(repo, snapshot1).getIndices().get(indexName).getShards().get(0).getStats();
    final int totalFilesInShard = firstSnapshotShardStatus.getTotalFileCount();
    assertThat(totalFilesInShard, greaterThan(0));
    final SnapshotStats secondSnapshotShardStatus = getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
    assertThat(secondSnapshotShardStatus.getTotalFileCount(), is(totalFilesInShard));
    assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), is(0));
}

Example 2 with SnapshotStats

use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats in project OpenSearch by opensearch-project.

the class BlobStoreIncrementalityIT method testForceMergeCausesFullSnapshot.

public void testForceMergeCausesFullSnapshot() throws Exception {
    internalCluster().startMasterOnlyNode();
    internalCluster().ensureAtLeastNumDataNodes(2);
    final String indexName = "test-index";
    createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build());
    ensureGreen(indexName);
    logger.info("--> adding some documents to test index and flush in between to get at least two segments");
    for (int j = 0; j < 2; j++) {
        final BulkRequest bulkRequest = new BulkRequest();
        for (int i = 0; i < scaledRandomIntBetween(1, 100); ++i) {
            bulkRequest.add(new IndexRequest(indexName).source("foo" + j, "bar" + i));
        }
        client().bulk(bulkRequest).get();
        flushAndRefresh(indexName);
    }
    final IndexStats indexStats = client().admin().indices().prepareStats(indexName).get().getIndex(indexName);
    assertThat(indexStats.getIndexShards().get(0).getPrimary().getSegments().getCount(), greaterThan(1L));
    final String snapshot1 = "snap-1";
    final String repo = "test-repo";
    createRepository(repo, "fs");
    logger.info("--> creating snapshot 1");
    client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get();
    logger.info("--> force merging down to a single segment");
    final ForceMergeResponse forceMergeResponse = client().admin().indices().prepareForceMerge(indexName).setMaxNumSegments(1).setFlush(true).get();
    assertThat(forceMergeResponse.getFailedShards(), is(0));
    final String snapshot2 = "snap-2";
    logger.info("--> creating snapshot 2");
    client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setIndices(indexName).setWaitForCompletion(true).get();
    logger.info("--> asserting that the two snapshots refer to different files in the repository");
    final SnapshotStats secondSnapshotShardStatus = getStats(repo, snapshot2).getIndices().get(indexName).getShards().get(0).getStats();
    assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0));
}

Example 3 with SnapshotStats

use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats in project OpenSearch by opensearch-project.

the class DedicatedClusterSnapshotRestoreIT method testSnapshotTotalAndIncrementalSizes.

public void testSnapshotTotalAndIncrementalSizes() throws Exception {
    final String indexName = "test-blocks-1";
    final String repositoryName = "repo-" + indexName;
    final String snapshot0 = "snapshot-0";
    final String snapshot1 = "snapshot-1";
    createIndex(indexName);
    int docs = between(10, 100);
    for (int i = 0; i < docs; i++) {
        client().prepareIndex(indexName).setSource("test", "init").execute().actionGet();
    }
    final Path repoPath = randomRepoPath();
    createRepository(repositoryName, "fs", repoPath);
    createFullSnapshot(repositoryName, snapshot0);
    SnapshotsStatusResponse response = clusterAdmin().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot0).get();
    List<SnapshotStatus> snapshots = response.getSnapshots();
    List<Path> snapshot0Files = scanSnapshotFolder(repoPath);
    assertThat(snapshots, hasSize(1));
    final int snapshot0FileCount = snapshot0Files.size();
    final long snapshot0FileSize = calculateTotalFilesSize(snapshot0Files);
    SnapshotStats stats = snapshots.get(0).getStats();
    final List<Path> snapshot0IndexMetaFiles = findRepoMetaBlobs(repoPath);
    // snapshotting a single index
    assertThat(snapshot0IndexMetaFiles, hasSize(1));
    assertThat(stats.getTotalFileCount(), greaterThanOrEqualTo(snapshot0FileCount));
    assertThat(stats.getTotalSize(), greaterThanOrEqualTo(snapshot0FileSize));
    assertThat(stats.getIncrementalFileCount(), equalTo(stats.getTotalFileCount()));
    assertThat(stats.getIncrementalSize(), equalTo(stats.getTotalSize()));
    assertThat(stats.getIncrementalFileCount(), equalTo(stats.getProcessedFileCount()));
    assertThat(stats.getIncrementalSize(), equalTo(stats.getProcessedSize()));
    // add few docs - less than initially
    docs = between(1, 5);
    for (int i = 0; i < docs; i++) {
        client().prepareIndex(indexName).setSource("test", "test" + i).execute().actionGet();
    }
    // create another snapshot
    // total size has to grow and has to be equal to files on fs
    createFullSnapshot(repositoryName, snapshot1);
    // drop 1st one to avoid miscalculation as snapshot reuses some files of prev snapshot
    assertAcked(startDeleteSnapshot(repositoryName, snapshot0).get());
    response = clusterAdmin().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot1).get();
    final List<Path> snapshot1Files = scanSnapshotFolder(repoPath);
    final List<Path> snapshot1IndexMetaFiles = findRepoMetaBlobs(repoPath);
    // The IndexMetadata did not change between snapshots, verify that no new redundant IndexMetaData was written to the repository
    assertThat(snapshot1IndexMetaFiles, is(snapshot0IndexMetaFiles));
    final int snapshot1FileCount = snapshot1Files.size();
    final long snapshot1FileSize = calculateTotalFilesSize(snapshot1Files);
    snapshots = response.getSnapshots();
    SnapshotStats anotherStats = snapshots.get(0).getStats();
    ArrayList<Path> snapshotFilesDiff = new ArrayList<>(snapshot1Files);
    snapshotFilesDiff.removeAll(snapshot0Files);
    assertThat(anotherStats.getIncrementalFileCount(), greaterThanOrEqualTo(snapshotFilesDiff.size()));
    assertThat(anotherStats.getIncrementalSize(), greaterThanOrEqualTo(calculateTotalFilesSize(snapshotFilesDiff)));
    assertThat(anotherStats.getIncrementalFileCount(), equalTo(anotherStats.getProcessedFileCount()));
    assertThat(anotherStats.getIncrementalSize(), equalTo(anotherStats.getProcessedSize()));
    assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize()));
    assertThat(stats.getTotalFileCount(), lessThan(anotherStats.getTotalFileCount()));
    assertThat(anotherStats.getTotalFileCount(), greaterThanOrEqualTo(snapshot1FileCount));
    assertThat(anotherStats.getTotalSize(), greaterThanOrEqualTo(snapshot1FileSize));
}

Example 4 with SnapshotStats

use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats in project OpenSearch by opensearch-project.

the class SnapshotStatusApisIT method testCorrectCountsForDoneShards.

/**
 * Tests the following sequence of steps:
 * 1. Start snapshot of two shards (both located on separate data nodes).
 * 2. Have one of the shards snapshot completely and the other block
 * 3. Restart the data node that completed its shard snapshot
 * 4. Make sure that snapshot status APIs show correct file-counts and -sizes
 *
 * @throws Exception on failure
 */
public void testCorrectCountsForDoneShards() throws Exception {
    final String indexOne = "index-1";
    final String indexTwo = "index-2";
    final List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
    final String dataNodeOne = dataNodes.get(0);
    final String dataNodeTwo = dataNodes.get(1);
    createIndex(indexOne, singleShardOneNode(dataNodeOne));
    index(indexOne, "_doc", "some_doc_id", "foo", "bar");
    createIndex(indexTwo, singleShardOneNode(dataNodeTwo));
    index(indexTwo, "_doc", "some_doc_id", "foo", "bar");
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    blockDataNode(repoName, dataNodeOne);
    final String snapshotOne = "snap-1";
    // restarting a data node below so using a master client here
    final ActionFuture<CreateSnapshotResponse> responseSnapshotOne = internalCluster().masterClient().admin().cluster().prepareCreateSnapshot(repoName, snapshotOne).setWaitForCompletion(true).execute();
    assertBusy(() -> {
        final SnapshotStatus snapshotStatusOne = getSnapshotStatus(repoName, snapshotOne);
        assertThat(snapshotStatusOne.getState(), is(SnapshotsInProgress.State.STARTED));
        final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatusOne, indexTwo);
        assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE));
        assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0));
        assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L));
    }, 30L, TimeUnit.SECONDS);
    final SnapshotStats snapshotShardStats = stateFirstShard(getSnapshotStatus(repoName, snapshotOne), indexTwo).getStats();
    final int totalFiles = snapshotShardStats.getTotalFileCount();
    final long totalFileSize = snapshotShardStats.getTotalSize();
    internalCluster().restartNode(dataNodeTwo);
    final SnapshotIndexShardStatus snapshotShardStateAfterNodeRestart = stateFirstShard(getSnapshotStatus(repoName, snapshotOne), indexTwo);
    assertThat(snapshotShardStateAfterNodeRestart.getStage(), is(SnapshotIndexShardStage.DONE));
    assertThat(snapshotShardStateAfterNodeRestart.getStats().getTotalFileCount(), equalTo(totalFiles));
    assertThat(snapshotShardStateAfterNodeRestart.getStats().getTotalSize(), equalTo(totalFileSize));
    unblockAllDataNodes(repoName);
    assertThat(responseSnapshotOne.get().getSnapshotInfo().state(), is(SnapshotState.SUCCESS));
    // indexing another document to the second index so it will do writes during the snapshot and we can block on those writes
    index(indexTwo, "_doc", "some_other_doc_id", "foo", "other_bar");
    blockDataNode(repoName, dataNodeTwo);
    final String snapshotTwo = "snap-2";
    final ActionFuture<CreateSnapshotResponse> responseSnapshotTwo = client().admin().cluster().prepareCreateSnapshot(repoName, snapshotTwo).setWaitForCompletion(true).execute();
    waitForBlock(dataNodeTwo, repoName, TimeValue.timeValueSeconds(30L));
    assertBusy(() -> {
        final SnapshotStatus snapshotStatusOne = getSnapshotStatus(repoName, snapshotOne);
        final SnapshotStatus snapshotStatusTwo = getSnapshotStatus(repoName, snapshotTwo);
        final SnapshotIndexShardStatus snapshotShardStateOne = stateFirstShard(snapshotStatusOne, indexOne);
        final SnapshotIndexShardStatus snapshotShardStateTwo = stateFirstShard(snapshotStatusTwo, indexOne);
        assertThat(snapshotShardStateOne.getStage(), is(SnapshotIndexShardStage.DONE));
        assertThat(snapshotShardStateTwo.getStage(), is(SnapshotIndexShardStage.DONE));
        final int totalFilesShardOne = snapshotShardStateOne.getStats().getTotalFileCount();
        final long totalSizeShardOne = snapshotShardStateOne.getStats().getTotalSize();
        assertThat(totalFilesShardOne, greaterThan(0));
        assertThat(totalSizeShardOne, greaterThan(0L));
        assertThat(totalFilesShardOne, equalTo(snapshotShardStateTwo.getStats().getTotalFileCount()));
        assertThat(totalSizeShardOne, equalTo(snapshotShardStateTwo.getStats().getTotalSize()));
        assertThat(snapshotShardStateTwo.getStats().getIncrementalFileCount(), equalTo(0));
        assertThat(snapshotShardStateTwo.getStats().getIncrementalSize(), equalTo(0L));
    }, 30L, TimeUnit.SECONDS);
    unblockAllDataNodes(repoName);
    assertThat(responseSnapshotTwo.get().getSnapshotInfo().state(), is(SnapshotState.SUCCESS));
}

Example 5 with SnapshotStats

use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats in project OpenSearch by opensearch-project.

the class SnapshotClientDocumentationIT method testSnapshotSnapshotsStatus.

public void testSnapshotSnapshotsStatus() throws IOException {
    RestHighLevelClient client = highLevelClient();
    createTestRepositories();
    createTestIndex();
    createTestSnapshots();
    // tag::snapshots-status-request
    SnapshotsStatusRequest request = new SnapshotsStatusRequest();
    // end::snapshots-status-request
    // tag::snapshots-status-request-repository
    // <1>
    request.repository(repositoryName);
    // end::snapshots-status-request-repository
    // tag::snapshots-status-request-snapshots
    String[] snapshots = new String[] { snapshotName };
    // <1>
    request.snapshots(snapshots);
    // end::snapshots-status-request-snapshots
    // tag::snapshots-status-request-ignoreUnavailable
    // <1>
    request.ignoreUnavailable(true);
    // end::snapshots-status-request-ignoreUnavailable
    // tag::snapshots-status-request-masterTimeout
    // <1>
    request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
    // <2>
    request.masterNodeTimeout("1m");
    // end::snapshots-status-request-masterTimeout
    // tag::snapshots-status-execute
    SnapshotsStatusResponse response = client.snapshot().status(request, RequestOptions.DEFAULT);
    // end::snapshots-status-execute
    // tag::snapshots-status-response
    List<SnapshotStatus> snapshotStatusesResponse = response.getSnapshots();
    // <1>
    SnapshotStatus snapshotStatus = snapshotStatusesResponse.get(0);
    // <2>
    SnapshotsInProgress.State snapshotState = snapshotStatus.getState();
    // <3>
    SnapshotStats shardStats = snapshotStatus.getIndices().get(indexName).getShards().get(0).getStats();
    // end::snapshots-status-response
    assertThat(snapshotStatusesResponse.size(), equalTo(1));
    assertThat(snapshotStatusesResponse.get(0).getSnapshot().getRepository(), equalTo(SnapshotClientDocumentationIT.repositoryName));
    assertThat(snapshotStatusesResponse.get(0).getSnapshot().getSnapshotId().getName(), equalTo(snapshotName));
    assertThat(snapshotState.completed(), equalTo(true));
}