Examples with RepositoryData org.opensearch.repositories.RepositoryData used on opensource projects

Example 31 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testSnapshotWithCorruptedShardIndexFile.

/**
 * Tests that a shard snapshot with a corrupted shard index file can still be used for restore and incremental snapshots.
 */
public void testSnapshotWithCorruptedShardIndexFile() throws Exception {
    final Client client = client();
    final Path repo = randomRepoPath();
    final String indexName = "test-idx";
    final int nDocs = randomIntBetween(1, 10);
    logger.info("-->  creating index [{}] with [{}] documents in it", indexName, nDocs);
    assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(1)));
    final IndexRequestBuilder[] documents = new IndexRequestBuilder[nDocs];
    for (int j = 0; j < nDocs; j++) {
        documents[j] = client.prepareIndex(indexName).setSource("foo", "bar");
    }
    indexRandom(true, documents);
    flushAndRefresh();
    createRepository("test-repo", "fs", repo);
    final String snapshot1 = "test-snap-1";
    final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", snapshot1);
    assertThat(snapshotInfo.indices(), hasSize(1));
    final RepositoryData repositoryData = getRepositoryData("test-repo");
    final Map<String, IndexId> indexIds = repositoryData.getIndices();
    assertThat(indexIds.size(), equalTo(1));
    final IndexId corruptedIndex = indexIds.get(indexName);
    final Path shardIndexFile = repo.resolve("indices").resolve(corruptedIndex.getId()).resolve("0").resolve("index-" + repositoryData.shardGenerations().getShardGen(corruptedIndex, 0));
    logger.info("-->  truncating shard index file [{}]", shardIndexFile);
    try (SeekableByteChannel outChan = Files.newByteChannel(shardIndexFile, StandardOpenOption.WRITE)) {
        outChan.truncate(randomInt(10));
    }
    logger.info("-->  verifying snapshot state for [{}]", snapshot1);
    List<SnapshotInfo> snapshotInfos = clusterAdmin().prepareGetSnapshots("test-repo").get().getSnapshots();
    assertThat(snapshotInfos.size(), equalTo(1));
    assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
    assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo(snapshot1));
    logger.info("-->  deleting index [{}]", indexName);
    assertAcked(client().admin().indices().prepareDelete(indexName));
    logger.info("-->  restoring snapshot [{}]", snapshot1);
    clusterAdmin().prepareRestoreSnapshot("test-repo", snapshot1).setRestoreGlobalState(randomBoolean()).setWaitForCompletion(true).get();
    ensureGreen();
    assertDocCount(indexName, nDocs);
    logger.info("-->  indexing [{}] more documents into [{}]", nDocs, indexName);
    for (int j = 0; j < nDocs; j++) {
        documents[j] = client.prepareIndex(indexName).setSource("foo2", "bar2");
    }
    indexRandom(true, documents);
    final String snapshot2 = "test-snap-2";
    logger.info("-->  creating snapshot [{}]", snapshot2);
    final SnapshotInfo snapshotInfo2 = clusterAdmin().prepareCreateSnapshot("test-repo", snapshot2).setWaitForCompletion(true).get().getSnapshotInfo();
    assertThat(snapshotInfo2.state(), equalTo(SnapshotState.PARTIAL));
    assertThat(snapshotInfo2.failedShards(), equalTo(1));
    assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo.totalShards() - 1));
    assertThat(snapshotInfo2.indices(), hasSize(1));
}

Example 32 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testRepairBrokenShardGenerations.

public void testRepairBrokenShardGenerations() throws Exception {
    final String repoName = "test-repo";
    final Path repoPath = randomRepoPath();
    createRepository(repoName, "fs", repoPath);
    final String oldVersionSnapshot = initWithSnapshotVersion(repoName, repoPath, SnapshotsService.OLD_SNAPSHOT_FORMAT);
    logger.info("--> recreating repository to clear caches");
    client().admin().cluster().prepareDeleteRepository(repoName).get();
    createRepository(repoName, "fs", repoPath);
    final String indexName = "test-index";
    createIndex(indexName);
    createFullSnapshot(repoName, "snapshot-1");
    startDeleteSnapshot(repoName, oldVersionSnapshot).get();
    logger.info("--> move shard level metadata to new generation and make RepositoryData point at an older generation");
    final IndexId indexId = getRepositoryData(repoName).resolveIndexId(indexName);
    final Path shardPath = repoPath.resolve("indices").resolve(indexId.getId()).resolve("0");
    final Path initialShardMetaPath = shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "0");
    assertFileExists(initialShardMetaPath);
    Files.move(initialShardMetaPath, shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + randomIntBetween(1, 1000)));
    final RepositoryData repositoryData1 = getRepositoryData(repoName);
    final Map<String, SnapshotId> snapshotIds = repositoryData1.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
    final RepositoryData brokenRepoData = new RepositoryData(repositoryData1.getGenId(), snapshotIds, snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData1::getSnapshotState)), snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData1::getVersion)), repositoryData1.getIndices().values().stream().collect(Collectors.toMap(Function.identity(), repositoryData1::getSnapshots)), ShardGenerations.builder().putAll(repositoryData1.shardGenerations()).put(indexId, 0, "0").build(), repositoryData1.indexMetaDataGenerations());
    Files.write(repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData1.getGenId()), BytesReference.toBytes(BytesReference.bytes(brokenRepoData.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))), StandardOpenOption.TRUNCATE_EXISTING);
    logger.info("--> recreating repository to clear caches");
    client().admin().cluster().prepareDeleteRepository(repoName).get();
    createRepository(repoName, "fs", repoPath);
    createFullSnapshot(repoName, "snapshot-2");
}

Example 33 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class ConcurrentSnapshotsIT method testConcurrentSnapshotWorksWithOldVersionRepo.

public void testConcurrentSnapshotWorksWithOldVersionRepo() throws Exception {
    internalCluster().startMasterOnlyNode();
    final String dataNode = internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    final Path repoPath = randomRepoPath();
    createRepository(repoName, "mock", Settings.builder().put(BlobStoreRepository.CACHE_REPOSITORY_DATA.getKey(), false).put("location", repoPath));
    initWithSnapshotVersion(repoName, repoPath, SnapshotsService.OLD_SNAPSHOT_FORMAT);
    createIndexWithContent("index-slow");
    final ActionFuture<CreateSnapshotResponse> createSlowFuture = startFullSnapshotBlockedOnDataNode("slow-snapshot", repoName, dataNode);
    final String dataNode2 = internalCluster().startDataOnlyNode();
    ensureStableCluster(3);
    final String indexFast = "index-fast";
    createIndexWithContent(indexFast, dataNode2, dataNode);
    final ActionFuture<CreateSnapshotResponse> createFastSnapshot = startFullSnapshot(repoName, "fast-snapshot");
    assertThat(createSlowFuture.isDone(), is(false));
    unblockNode(repoName, dataNode);
    assertSuccessful(createFastSnapshot);
    assertSuccessful(createSlowFuture);
    final RepositoryData repositoryData = getRepositoryData(repoName);
    assertThat(repositoryData.shardGenerations(), is(ShardGenerations.EMPTY));
}

Example 34 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class ConcurrentSnapshotsIT method testDeletesAreBatched.

public void testDeletesAreBatched() throws Exception {
    internalCluster().startMasterOnlyNode();
    final String dataNode = internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    createIndex("foo");
    ensureGreen();
    final int numSnapshots = randomIntBetween(1, 4);
    final PlainActionFuture<Collection<CreateSnapshotResponse>> allSnapshotsDone = PlainActionFuture.newFuture();
    final ActionListener<CreateSnapshotResponse> snapshotsListener = new GroupedActionListener<>(allSnapshotsDone, numSnapshots);
    final Collection<String> snapshotNames = new HashSet<>();
    for (int i = 0; i < numSnapshots; i++) {
        final String snapshot = "snap-" + i;
        snapshotNames.add(snapshot);
        client().admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true).execute(snapshotsListener);
    }
    final Collection<CreateSnapshotResponse> snapshotResponses = allSnapshotsDone.get();
    for (CreateSnapshotResponse snapshotResponse : snapshotResponses) {
        assertThat(snapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS));
    }
    createIndexWithContent("index-slow");
    final ActionFuture<CreateSnapshotResponse> createSlowFuture = startFullSnapshotBlockedOnDataNode("blocked-snapshot", repoName, dataNode);
    final Collection<StepListener<AcknowledgedResponse>> deleteFutures = new ArrayList<>();
    while (snapshotNames.isEmpty() == false) {
        final Collection<String> toDelete = randomSubsetOf(snapshotNames);
        if (toDelete.isEmpty()) {
            continue;
        }
        snapshotNames.removeAll(toDelete);
        final StepListener<AcknowledgedResponse> future = new StepListener<>();
        client().admin().cluster().prepareDeleteSnapshot(repoName, toDelete.toArray(Strings.EMPTY_ARRAY)).execute(future);
        deleteFutures.add(future);
    }
    assertThat(createSlowFuture.isDone(), is(false));
    final long repoGenAfterInitialSnapshots = getRepositoryData(repoName).getGenId();
    assertThat(repoGenAfterInitialSnapshots, is(numSnapshots - 1L));
    unblockNode(repoName, dataNode);
    final SnapshotInfo slowSnapshotInfo = assertSuccessful(createSlowFuture);
    logger.info("--> waiting for batched deletes to finish");
    final PlainActionFuture<Collection<AcknowledgedResponse>> allDeletesDone = new PlainActionFuture<>();
    final ActionListener<AcknowledgedResponse> deletesListener = new GroupedActionListener<>(allDeletesDone, deleteFutures.size());
    for (StepListener<AcknowledgedResponse> deleteFuture : deleteFutures) {
        deleteFuture.whenComplete(deletesListener::onResponse, deletesListener::onFailure);
    }
    allDeletesDone.get();
    logger.info("--> verifying repository state");
    final RepositoryData repositoryDataAfterDeletes = getRepositoryData(repoName);
    // One increment for snapshot, one for all the deletes
    assertThat(repositoryDataAfterDeletes.getGenId(), is(repoGenAfterInitialSnapshots + 2));
    assertThat(repositoryDataAfterDeletes.getSnapshotIds(), contains(slowSnapshotInfo.snapshotId()));
}

Example 35 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class ConcurrentSnapshotsIT method testBackToBackQueuedDeletes.

public void testBackToBackQueuedDeletes() throws Exception {
    final String masterName = internalCluster().startMasterOnlyNode();
    internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    createIndexWithContent("index-test");
    final List<String> snapshots = createNSnapshots(repoName, 2);
    final String snapshotOne = snapshots.get(0);
    final String snapshotTwo = snapshots.get(1);
    final ActionFuture<AcknowledgedResponse> deleteSnapshotOne = startAndBlockOnDeleteSnapshot(repoName, snapshotOne);
    final ActionFuture<AcknowledgedResponse> deleteSnapshotTwo = startDeleteSnapshot(repoName, snapshotTwo);
    awaitNDeletionsInProgress(2);
    unblockNode(repoName, masterName);
    assertAcked(deleteSnapshotOne.get());
    assertAcked(deleteSnapshotTwo.get());
    final RepositoryData repositoryData = getRepositoryData(repoName);
    assertThat(repositoryData.getSnapshotIds(), empty());
    // Two snapshots and two distinct delete operations move us 4 steps from -1 to 3
    assertThat(repositoryData.getGenId(), is(3L));
}