Examples with IndexId org.opensearch.repositories.IndexId used on opensource projects

Example 1 with IndexId

use of org.opensearch.repositories.IndexId in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method buildSimpleSnapshotInfos.

private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve, final RepositoryData repositoryData, final List<SnapshotInfo> currentSnapshots) {
    List<SnapshotInfo> snapshotInfos = new ArrayList<>();
    for (SnapshotInfo snapshotInfo : currentSnapshots) {
        if (toResolve.remove(snapshotInfo.snapshotId())) {
            snapshotInfos.add(snapshotInfo.basic());
        }
    }
    Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>();
    for (IndexId indexId : repositoryData.getIndices().values()) {
        for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
            if (toResolve.contains(snapshotId)) {
                snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()).add(indexId.getName());
            }
        }
    }
    for (SnapshotId snapshotId : toResolve) {
        final List<String> indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList());
        CollectionUtil.timSort(indices);
        snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), repositoryData.getSnapshotState(snapshotId)));
    }
    CollectionUtil.timSort(snapshotInfos);
    return unmodifiableList(snapshotInfos);
}

Example 2 with IndexId

use of org.opensearch.repositories.IndexId in project OpenSearch by opensearch-project.

the class TransportSnapshotsStatusAction method snapshotShards.

/**
 * Returns status of shards currently finished snapshots
 * <p>
 * This method is executed on master node and it's complimentary to the
 * {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
 * returns similar information but for already finished snapshots.
 * </p>
 *
 * @param repositoryName  repository name
 * @param snapshotInfo    snapshot info
 * @return map of shard id to snapshot status
 */
private Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName, final RepositoryData repositoryData, final SnapshotInfo snapshotInfo) throws IOException {
    final Repository repository = repositoriesService.repository(repositoryName);
    final Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
    for (String index : snapshotInfo.indices()) {
        IndexId indexId = repositoryData.resolveIndexId(index);
        IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotInfo.snapshotId(), indexId);
        if (indexMetadata != null) {
            int numberOfShards = indexMetadata.getNumberOfShards();
            for (int i = 0; i < numberOfShards; i++) {
                ShardId shardId = new ShardId(indexMetadata.getIndex(), i);
                SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                if (shardFailure != null) {
                    shardStatus.put(shardId, IndexShardSnapshotStatus.newFailed(shardFailure.reason()));
                } else {
                    final IndexShardSnapshotStatus shardSnapshotStatus;
                    if (snapshotInfo.state() == SnapshotState.FAILED) {
                        // If the snapshot failed, but the shard's snapshot does
                        // not have an exception, it means that partial snapshots
                        // were disabled and in this case, the shard snapshot will
                        // *not* have any metadata, so attempting to read the shard
                        // snapshot status will throw an exception. Instead, we create
                        // a status for the shard to indicate that the shard snapshot
                        // could not be taken due to partial being set to false.
                        shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped");
                    } else {
                        shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), indexId, shardId);
                    }
                    shardStatus.put(shardId, shardSnapshotStatus);
                }
            }
        }
    }
    return unmodifiableMap(shardStatus);
}

Example 3 with IndexId

use of org.opensearch.repositories.IndexId in project OpenSearch by opensearch-project.

the class SharedClusterSnapshotRestoreIT method testRestoreSnapshotWithCorruptedIndexMetadata.

/**
 * Tests that a snapshot of multiple indices including one with a corrupted index metadata
 * file can still be used to restore the non corrupted indices
 */
public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception {
    final Client client = client();
    final Path repo = randomRepoPath();
    final int nbIndices = randomIntBetween(2, 3);
    final Map<String, Integer> nbDocsPerIndex = new HashMap<>();
    for (int i = 0; i < nbIndices; i++) {
        String indexName = "test-idx-" + i;
        assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(Math.min(2, numberOfShards()))));
        int nbDocs = randomIntBetween(1, 10);
        nbDocsPerIndex.put(indexName, nbDocs);
        IndexRequestBuilder[] documents = new IndexRequestBuilder[nbDocs];
        for (int j = 0; j < nbDocs; j++) {
            documents[j] = client.prepareIndex(indexName).setSource("foo", "bar");
        }
        indexRandom(true, documents);
    }
    flushAndRefresh();
    createRepository("test-repo", "fs", repo);
    final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap");
    assertThat(snapshotInfo.indices(), hasSize(nbIndices));
    final RepositoryData repositoryData = getRepositoryData("test-repo");
    final Map<String, IndexId> indexIds = repositoryData.getIndices();
    assertThat(indexIds.size(), equalTo(nbIndices));
    // Choose a random index from the snapshot
    final IndexId corruptedIndex = randomFrom(indexIds.values());
    final Path indexMetadataPath = repo.resolve("indices").resolve(corruptedIndex.getId()).resolve("meta-" + repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotInfo.snapshotId(), corruptedIndex) + ".dat");
    // Truncate the index metadata file
    try (SeekableByteChannel outChan = Files.newByteChannel(indexMetadataPath, StandardOpenOption.WRITE)) {
        outChan.truncate(randomInt(10));
    }
    List<SnapshotInfo> snapshotInfos = clusterAdmin().prepareGetSnapshots("test-repo").get().getSnapshots();
    assertThat(snapshotInfos.size(), equalTo(1));
    assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
    assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
    assertAcked(client().admin().indices().prepareDelete(nbDocsPerIndex.keySet().toArray(new String[nbDocsPerIndex.size()])));
    Predicate<String> isRestorableIndex = index -> corruptedIndex.getName().equals(index) == false;
    clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setIndices(nbDocsPerIndex.keySet().stream().filter(isRestorableIndex).toArray(String[]::new)).setRestoreGlobalState(randomBoolean()).setWaitForCompletion(true).get();
    ensureGreen();
    for (Map.Entry<String, Integer> entry : nbDocsPerIndex.entrySet()) {
        if (isRestorableIndex.test(entry.getKey())) {
            assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().longValue());
        }
    }
    assertAcked(startDeleteSnapshot("test-repo", snapshotInfo.snapshotId().getName()).get());
}

Example 4 with IndexId

use of org.opensearch.repositories.IndexId in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testHandleSnapshotErrorWithBwCFormat.

public void testHandleSnapshotErrorWithBwCFormat() throws IOException, ExecutionException, InterruptedException {
    final String repoName = "test-repo";
    final Path repoPath = randomRepoPath();
    createRepository(repoName, "fs", repoPath);
    final String oldVersionSnapshot = initWithSnapshotVersion(repoName, repoPath, SnapshotsService.OLD_SNAPSHOT_FORMAT);
    logger.info("--> recreating repository to clear caches");
    client().admin().cluster().prepareDeleteRepository(repoName).get();
    createRepository(repoName, "fs", repoPath);
    final String indexName = "test-index";
    createIndex(indexName);
    createFullSnapshot(repoName, "snapshot-1");
    // In the old metadata version the shard level metadata could be moved to the next generation for all sorts of reasons, this should
    // not break subsequent repository operations
    logger.info("--> move shard level metadata to new generation");
    final IndexId indexId = getRepositoryData(repoName).resolveIndexId(indexName);
    final Path shardPath = repoPath.resolve("indices").resolve(indexId.getId()).resolve("0");
    final Path initialShardMetaPath = shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "0");
    assertFileExists(initialShardMetaPath);
    Files.move(initialShardMetaPath, shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "1"));
    startDeleteSnapshot(repoName, oldVersionSnapshot).get();
    createFullSnapshot(repoName, "snapshot-2");
}

Example 5 with IndexId

use of org.opensearch.repositories.IndexId in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testDeleteSnapshotWithMissingIndexAndShardMetadata.

public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception {
    Client client = client();
    Path repo = randomRepoPath();
    createRepository("test-repo", "fs", repo);
    final String[] indices = { "test-idx-1", "test-idx-2" };
    createIndex(indices);
    logger.info("--> indexing some data");
    indexRandom(true, client().prepareIndex("test-idx-1").setSource("foo", "bar"), client().prepareIndex("test-idx-2").setSource("foo", "bar"));
    logger.info("--> creating snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices(indices).get();
    final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
    assertThat(snapshotInfo.successfulShards(), greaterThan(0));
    assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
    final Map<String, IndexId> indexIds = getRepositoryData("test-repo").getIndices();
    final Path indicesPath = repo.resolve("indices");
    logger.info("--> delete index metadata and shard metadata");
    for (String index : indices) {
        Path shardZero = indicesPath.resolve(indexIds.get(index).getId()).resolve("0");
        if (randomBoolean()) {
            Files.delete(shardZero.resolve("index-" + getRepositoryData("test-repo").shardGenerations().getShardGen(indexIds.get(index), 0)));
        }
        Files.delete(shardZero.resolve("snap-" + snapshotInfo.snapshotId().getUUID() + ".dat"));
    }
    startDeleteSnapshot("test-repo", "test-snap-1").get();
    logger.info("--> make sure snapshot doesn't exist");
    expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1").get());
    for (String index : indices) {
        assertTrue(Files.notExists(indicesPath.resolve(indexIds.get(index).getId())));
    }
}