use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus in project OpenSearch by opensearch-project.
the class DedicatedClusterSnapshotRestoreIT method testSnapshotWithDateMath.
public void testSnapshotWithDateMath() {
final String repo = "repo";
final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
final String snapshotName = "<snapshot-{now/d}>";
logger.info("--> creating repository");
assertAcked(clusterAdmin().preparePutRepository(repo).setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())));
final String expression1 = nameExpressionResolver.resolveDateMathExpression(snapshotName);
logger.info("--> creating date math snapshot");
createFullSnapshot(repo, snapshotName);
// snapshot could be taken before or after a day rollover
final String expression2 = nameExpressionResolver.resolveDateMathExpression(snapshotName);
SnapshotsStatusResponse response = clusterAdmin().prepareSnapshotStatus(repo).setSnapshots(Sets.newHashSet(expression1, expression2).toArray(Strings.EMPTY_ARRAY)).setIgnoreUnavailable(true).get();
List<SnapshotStatus> snapshots = response.getSnapshots();
assertThat(snapshots, hasSize(1));
assertThat(snapshots.get(0).getState().completed(), equalTo(true));
}
use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus in project OpenSearch by opensearch-project.
the class DedicatedClusterSnapshotRestoreIT method testSnapshotTotalAndIncrementalSizes.
public void testSnapshotTotalAndIncrementalSizes() throws Exception {
final String indexName = "test-blocks-1";
final String repositoryName = "repo-" + indexName;
final String snapshot0 = "snapshot-0";
final String snapshot1 = "snapshot-1";
createIndex(indexName);
int docs = between(10, 100);
for (int i = 0; i < docs; i++) {
client().prepareIndex(indexName).setSource("test", "init").execute().actionGet();
}
final Path repoPath = randomRepoPath();
createRepository(repositoryName, "fs", repoPath);
createFullSnapshot(repositoryName, snapshot0);
SnapshotsStatusResponse response = clusterAdmin().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot0).get();
List<SnapshotStatus> snapshots = response.getSnapshots();
List<Path> snapshot0Files = scanSnapshotFolder(repoPath);
assertThat(snapshots, hasSize(1));
final int snapshot0FileCount = snapshot0Files.size();
final long snapshot0FileSize = calculateTotalFilesSize(snapshot0Files);
SnapshotStats stats = snapshots.get(0).getStats();
final List<Path> snapshot0IndexMetaFiles = findRepoMetaBlobs(repoPath);
// snapshotting a single index
assertThat(snapshot0IndexMetaFiles, hasSize(1));
assertThat(stats.getTotalFileCount(), greaterThanOrEqualTo(snapshot0FileCount));
assertThat(stats.getTotalSize(), greaterThanOrEqualTo(snapshot0FileSize));
assertThat(stats.getIncrementalFileCount(), equalTo(stats.getTotalFileCount()));
assertThat(stats.getIncrementalSize(), equalTo(stats.getTotalSize()));
assertThat(stats.getIncrementalFileCount(), equalTo(stats.getProcessedFileCount()));
assertThat(stats.getIncrementalSize(), equalTo(stats.getProcessedSize()));
// add few docs - less than initially
docs = between(1, 5);
for (int i = 0; i < docs; i++) {
client().prepareIndex(indexName).setSource("test", "test" + i).execute().actionGet();
}
// create another snapshot
// total size has to grow and has to be equal to files on fs
createFullSnapshot(repositoryName, snapshot1);
// drop 1st one to avoid miscalculation as snapshot reuses some files of prev snapshot
assertAcked(startDeleteSnapshot(repositoryName, snapshot0).get());
response = clusterAdmin().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot1).get();
final List<Path> snapshot1Files = scanSnapshotFolder(repoPath);
final List<Path> snapshot1IndexMetaFiles = findRepoMetaBlobs(repoPath);
// The IndexMetadata did not change between snapshots, verify that no new redundant IndexMetaData was written to the repository
assertThat(snapshot1IndexMetaFiles, is(snapshot0IndexMetaFiles));
final int snapshot1FileCount = snapshot1Files.size();
final long snapshot1FileSize = calculateTotalFilesSize(snapshot1Files);
snapshots = response.getSnapshots();
SnapshotStats anotherStats = snapshots.get(0).getStats();
ArrayList<Path> snapshotFilesDiff = new ArrayList<>(snapshot1Files);
snapshotFilesDiff.removeAll(snapshot0Files);
assertThat(anotherStats.getIncrementalFileCount(), greaterThanOrEqualTo(snapshotFilesDiff.size()));
assertThat(anotherStats.getIncrementalSize(), greaterThanOrEqualTo(calculateTotalFilesSize(snapshotFilesDiff)));
assertThat(anotherStats.getIncrementalFileCount(), equalTo(anotherStats.getProcessedFileCount()));
assertThat(anotherStats.getIncrementalSize(), equalTo(anotherStats.getProcessedSize()));
assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize()));
assertThat(stats.getTotalFileCount(), lessThan(anotherStats.getTotalFileCount()));
assertThat(anotherStats.getTotalFileCount(), greaterThanOrEqualTo(snapshot1FileCount));
assertThat(anotherStats.getTotalSize(), greaterThanOrEqualTo(snapshot1FileSize));
}
use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus in project OpenSearch by opensearch-project.
the class SharedClusterSnapshotRestoreIT method testDataFileFailureDuringSnapshot.
public void testDataFileFailureDuringSnapshot() throws Exception {
disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
createRepository("test-repo", "mock", Settings.builder().put("location", randomRepoPath()).put("random", randomAlphaOfLength(10)).put("random_data_file_io_exception_rate", 0.3));
createIndexWithRandomDocs("test-idx", 100);
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
if (createSnapshotResponse.getSnapshotInfo().totalShards() == createSnapshotResponse.getSnapshotInfo().successfulShards()) {
logger.info("--> no failures");
// If we are here, that means we didn't have any failures, let's check it
assertThat(getFailureCount("test-repo"), equalTo(0L));
} else {
logger.info("--> some failures");
assertThat(getFailureCount("test-repo"), greaterThan(0L));
assertThat(createSnapshotResponse.getSnapshotInfo().shardFailures().size(), greaterThan(0));
for (SnapshotShardFailure shardFailure : createSnapshotResponse.getSnapshotInfo().shardFailures()) {
assertThat(shardFailure.nodeId(), notNullValue());
assertThat(shardFailure.index(), equalTo("test-idx"));
}
SnapshotInfo snapshotInfo = getSnapshot("test-repo", "test-snap");
assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL));
assertThat(snapshotInfo.shardFailures().size(), greaterThan(0));
assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards()));
// Verify that snapshot status also contains the same failures
SnapshotsStatusResponse snapshotsStatusResponse = clusterAdmin().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").get();
assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
assertThat(snapshotStatus.getIndices().size(), equalTo(1));
SnapshotIndexStatus indexStatus = snapshotStatus.getIndices().get("test-idx");
assertThat(indexStatus, notNullValue());
assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(snapshotInfo.failedShards()));
assertThat(indexStatus.getShardsStats().getDoneShards(), equalTo(snapshotInfo.successfulShards()));
assertThat(indexStatus.getShards().size(), equalTo(snapshotInfo.totalShards()));
int numberOfFailures = 0;
for (SnapshotIndexShardStatus shardStatus : indexStatus.getShards().values()) {
if (shardStatus.getStage() == SnapshotIndexShardStage.FAILURE) {
assertThat(shardStatus.getFailure(), notNullValue());
numberOfFailures++;
} else {
assertThat(shardStatus.getFailure(), nullValue());
}
}
assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(numberOfFailures));
}
}
use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus in project OpenSearch by opensearch-project.
the class SharedClusterSnapshotRestoreIT method testSnapshotMoreThanOnce.
public void testSnapshotMoreThanOnce() throws InterruptedException {
Client client = client();
createRepository("test-repo", "fs");
// only one shard
final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
assertAcked(prepareCreate("test").setSettings(indexSettings));
ensureGreen();
indexRandomDocs("test", randomIntBetween(10, 100));
assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get());
createSnapshot("test-repo", "test", Collections.singletonList("test"));
assertThat(getSnapshot("test-repo", "test").state(), equalTo(SnapshotState.SUCCESS));
{
SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test").get().getSnapshots().get(0);
List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
for (SnapshotIndexShardStatus status : shards) {
assertThat(status.getStats().getProcessedFileCount(), greaterThan(1));
}
}
createSnapshot("test-repo", "test-1", Collections.singletonList("test"));
assertThat(getSnapshot("test-repo", "test-1").state(), equalTo(SnapshotState.SUCCESS));
{
SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-1").get().getSnapshots().get(0);
List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
for (SnapshotIndexShardStatus status : shards) {
assertThat(status.getStats().getProcessedFileCount(), equalTo(0));
}
}
client().prepareDelete("test", "1").get();
createSnapshot("test-repo", "test-2", Collections.singletonList("test"));
assertThat(getSnapshot("test-repo", "test-2").state(), equalTo(SnapshotState.SUCCESS));
{
SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-2").get().getSnapshots().get(0);
Settings settings = client.admin().indices().prepareGetSettings("test").get().getIndexToSettings().get("test");
List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
for (SnapshotIndexShardStatus status : shards) {
// we flush before the snapshot such that we have to process the segments_N files plus the .del file
// soft-delete generates DV files.
assertThat(status.getStats().getProcessedFileCount(), greaterThan(2));
}
}
}
use of org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus in project OpenSearch by opensearch-project.
the class SharedClusterSnapshotRestoreIT method testSnapshotStatus.
public void testSnapshotStatus() throws Exception {
Client client = client();
createRepository("test-repo", "mock", Settings.builder().put("location", randomRepoPath()).put("random", randomAlphaOfLength(10)).put("wait_after_unblock", 200));
// Create index on 2 nodes and make sure each node has a primary by setting no replicas
assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_replicas", 0)));
indexRandomDocs("test-idx", 100);
// Pick one node and block it
String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
logger.info("--> snapshot");
client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIncludeGlobalState(false).setIndices("test-idx").get();
logger.info("--> waiting for block to kick in");
waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
logger.info("--> execution was blocked on node [{}], checking snapshot status with specified repository and snapshot", blockedNode);
SnapshotsStatusResponse response = client.admin().cluster().prepareSnapshotStatus("test-repo").execute().actionGet();
assertThat(response.getSnapshots().size(), equalTo(1));
SnapshotStatus snapshotStatus = response.getSnapshots().get(0);
assertThat(snapshotStatus.getState(), equalTo(State.STARTED));
assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
// We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
if (shardStatus.getStage() == SnapshotIndexShardStage.STARTED) {
assertThat(shardStatus.getNodeId(), notNullValue());
}
}
logger.info("--> checking snapshot status for all currently running and snapshot with empty repository");
response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
assertThat(response.getSnapshots().size(), equalTo(1));
snapshotStatus = response.getSnapshots().get(0);
assertThat(snapshotStatus.getState(), equalTo(State.STARTED));
assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
// We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
if (shardStatus.getStage() == SnapshotIndexShardStage.STARTED) {
assertThat(shardStatus.getNodeId(), notNullValue());
}
}
logger.info("--> checking that _current returns the currently running snapshot");
GetSnapshotsResponse getResponse = client.admin().cluster().prepareGetSnapshots("test-repo").setCurrentSnapshot().execute().actionGet();
assertThat(getResponse.getSnapshots().size(), equalTo(1));
SnapshotInfo snapshotInfo = getResponse.getSnapshots().get(0);
assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS));
logger.info("--> unblocking blocked node");
unblockNode("test-repo", blockedNode);
snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600));
logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
logger.info("--> done");
logger.info("--> checking snapshot status again after snapshot is done");
response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").execute().actionGet();
snapshotStatus = response.getSnapshots().get(0);
assertThat(snapshotStatus.getIndices().size(), equalTo(1));
assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
SnapshotIndexStatus indexStatus = snapshotStatus.getIndices().get("test-idx");
assertThat(indexStatus, notNullValue());
assertThat(indexStatus.getShardsStats().getInitializingShards(), equalTo(0));
assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(snapshotInfo.failedShards()));
assertThat(indexStatus.getShardsStats().getDoneShards(), equalTo(snapshotInfo.successfulShards()));
assertThat(indexStatus.getShards().size(), equalTo(snapshotInfo.totalShards()));
logger.info("--> checking snapshot status after it is done with empty repository");
response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
assertThat(response.getSnapshots().size(), equalTo(0));
logger.info("--> checking that _current no longer returns the snapshot");
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("_current").execute().actionGet().getSnapshots().isEmpty(), equalTo(true));
// test that getting an unavailable snapshot status throws an exception if ignoreUnavailable is false on the request
SnapshotMissingException ex = expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-doesnt-exist").get());
assertEquals("[test-repo:test-snap-doesnt-exist] is missing", ex.getMessage());
// test that getting an unavailable snapshot status does not throw an exception if ignoreUnavailable is true on the request
response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-doesnt-exist").setIgnoreUnavailable(true).get();
assertTrue(response.getSnapshots().isEmpty());
// test getting snapshot status for available and unavailable snapshots where ignoreUnavailable is true
// (available one should be returned)
response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap", "test-snap-doesnt-exist").setIgnoreUnavailable(true).get();
assertEquals(1, response.getSnapshots().size());
assertEquals("test-snap", response.getSnapshots().get(0).getSnapshot().getSnapshotId().getName());
}
Aggregations