Examples with RepositoryData org.opensearch.repositories.RepositoryData used on opensource projects

Example 1 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class ConcurrentSnapshotsIT method testMasterFailoverAndMultipleQueuedUpSnapshotsAcrossTwoRepos.

public void testMasterFailoverAndMultipleQueuedUpSnapshotsAcrossTwoRepos() throws Exception {
    disableRepoConsistencyCheck("This test corrupts the repository on purpose");
    internalCluster().startMasterOnlyNodes(3, LARGE_SNAPSHOT_POOL_SETTINGS);
    final String dataNode = internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    final String otherRepoName = "other-test-repo";
    final Path repoPath = randomRepoPath();
    createRepository(repoName, "mock", repoPath);
    createRepository(otherRepoName, "mock");
    createIndexWithContent("index-one");
    createNSnapshots(repoName, randomIntBetween(2, 5));
    final int countOtherRepo = randomIntBetween(2, 5);
    createNSnapshots(otherRepoName, countOtherRepo);
    corruptIndexN(repoPath, getRepositoryData(repoName).getGenId());
    blockMasterFromFinalizingSnapshotOnIndexFile(repoName);
    blockMasterFromFinalizingSnapshotOnIndexFile(otherRepoName);
    client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-blocked-1").setWaitForCompletion(false).get();
    client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-blocked-2").setWaitForCompletion(false).get();
    client().admin().cluster().prepareCreateSnapshot(otherRepoName, "snapshot-other-blocked-1").setWaitForCompletion(false).get();
    client().admin().cluster().prepareCreateSnapshot(otherRepoName, "snapshot-other-blocked-2").setWaitForCompletion(false).get();
    awaitNumberOfSnapshotsInProgress(4);
    final String initialMaster = internalCluster().getMasterName();
    waitForBlock(initialMaster, repoName, TimeValue.timeValueSeconds(30L));
    waitForBlock(initialMaster, otherRepoName, TimeValue.timeValueSeconds(30L));
    internalCluster().stopCurrentMasterNode();
    ensureStableCluster(3, dataNode);
    awaitNoMoreRunningOperations();
    final RepositoryData repositoryData = getRepositoryData(otherRepoName);
    assertThat(repositoryData.getSnapshotIds(), hasSize(countOtherRepo + 2));
}

Example 2 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method buildSimpleSnapshotInfos.

private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve, final RepositoryData repositoryData, final List<SnapshotInfo> currentSnapshots) {
    List<SnapshotInfo> snapshotInfos = new ArrayList<>();
    for (SnapshotInfo snapshotInfo : currentSnapshots) {
        if (toResolve.remove(snapshotInfo.snapshotId())) {
            snapshotInfos.add(snapshotInfo.basic());
        }
    }
    Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>();
    for (IndexId indexId : repositoryData.getIndices().values()) {
        for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
            if (toResolve.contains(snapshotId)) {
                snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()).add(indexId.getName());
            }
        }
    }
    for (SnapshotId snapshotId : toResolve) {
        final List<String> indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList());
        CollectionUtil.timSort(indices);
        snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), repositoryData.getSnapshotState(snapshotId)));
    }
    CollectionUtil.timSort(snapshotInfos);
    return unmodifiableList(snapshotInfos);
}

Example 3 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class TransportGetSnapshotsAction method masterOperation.

@Override
protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
    try {
        final String repository = request.repository();
        final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
        for (SnapshotInfo snapshotInfo : sortedCurrentSnapshots(snapshotsInProgress, repository)) {
            SnapshotId snapshotId = snapshotInfo.snapshotId();
            allSnapshotIds.put(snapshotId.getName(), snapshotId);
            currentSnapshots.add(snapshotInfo);
        }
        final RepositoryData repositoryData;
        if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
            repositoryData = PlainActionFuture.get(fut -> repositoriesService.getRepositoryData(repository, fut));
            for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
                allSnapshotIds.put(snapshotId.getName(), snapshotId);
            }
        } else {
            repositoryData = null;
        }
        final Set<SnapshotId> toResolve = new HashSet<>();
        if (isAllSnapshots(request.snapshots())) {
            toResolve.addAll(allSnapshotIds.values());
        } else {
            for (String snapshotOrPattern : request.snapshots()) {
                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
                    } else if (request.ignoreUnavailable() == false) {
                        throw new SnapshotMissingException(repository, snapshotOrPattern);
                    }
                } else {
                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
                            toResolve.add(entry.getValue());
                        }
                    }
                }
            }
            if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
                throw new SnapshotMissingException(repository, request.snapshots()[0]);
            }
        }
        final List<SnapshotInfo> snapshotInfos;
        if (request.verbose()) {
            snapshotInfos = snapshots(snapshotsInProgress, repository, new ArrayList<>(toResolve), request.ignoreUnavailable());
        } else {
            if (repositoryData != null) {
                // want non-current snapshots as well, which are found in the repository data
                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
            } else {
                // only want current snapshots
                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
                CollectionUtil.timSort(snapshotInfos);
            }
        }
        listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
    } catch (Exception e) {
        listener.onFailure(e);
    }
}

Example 4 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testSnapshotRecovery.

public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    assertAcked(client().admin().cluster().preparePutRepository(REPO_NAME).setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false)).get());
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).setIndices(INDEX_NAME).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
    logger.info("--> restore");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).execute().actionGet();
    int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
    assertThat(totalShards, greaterThan(0));
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    Repository repository = internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(REPO_NAME);
    final RepositoryData repositoryData = PlainActionFuture.get(repository::getRepositoryData);
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(totalShards));
        for (RecoveryState recoveryState : recoveryStates) {
            SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(((SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()), Version.CURRENT, repositoryData.resolveIndexId(INDEX_NAME));
            assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}

Example 5 with RepositoryData

use of org.opensearch.repositories.RepositoryData in project OpenSearch by opensearch-project.

the class TransportCleanupRepositoryAction method cleanupRepo.

/**
 * Runs cleanup operations on the given repository.
 * @param repositoryName Repository to clean up
 * @param listener Listener for cleanup result
 */
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
    final Repository repository = repositoriesService.repository(repositoryName);
    if (repository instanceof BlobStoreRepository == false) {
        listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
        return;
    }
    final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repository.getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final long repositoryStateId = repositoryData.getGenId();
        logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
        clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

            private boolean startedCleanup = false;

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
                if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
                }
                final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                if (snapshots.entries().isEmpty() == false) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
                }
                return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                after(e, null);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                startedCleanup = true;
                logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
            }

            private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
                if (failure == null) {
                    logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
                } else {
                    logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                }
                assert failure != null || result != null;
                if (startedCleanup == false) {
                    logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
                    listener.onFailure(failure);
                    return;
                }
                clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        return removeInProgressCleanup(currentState);
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        if (failure != null) {
                            e.addSuppressed(failure);
                        }
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (failure == null) {
                            logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
                            listener.onResponse(result);
                        } else {
                            logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                            listener.onFailure(failure);
                        }
                    }
                });
            }
        });
    }, listener::onFailure);
}