Examples with RepositoryCleanupResult org.opensearch.repositories.RepositoryCleanupResult used on opensource projects

Example 1 with RepositoryCleanupResult

use of org.opensearch.repositories.RepositoryCleanupResult in project OpenSearch by opensearch-project.

the class TransportCleanupRepositoryAction method cleanupRepo.

/**
 * Runs cleanup operations on the given repository.
 * @param repositoryName Repository to clean up
 * @param listener Listener for cleanup result
 */
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
    final Repository repository = repositoriesService.repository(repositoryName);
    if (repository instanceof BlobStoreRepository == false) {
        listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
        return;
    }
    final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repository.getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final long repositoryStateId = repositoryData.getGenId();
        logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
        clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

            private boolean startedCleanup = false;

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
                if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
                }
                final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                if (snapshots.entries().isEmpty() == false) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
                }
                return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                after(e, null);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                startedCleanup = true;
                logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
            }

            private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
                if (failure == null) {
                    logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
                } else {
                    logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                }
                assert failure != null || result != null;
                if (startedCleanup == false) {
                    logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
                    listener.onFailure(failure);
                    return;
                }
                clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        return removeInProgressCleanup(currentState);
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        if (failure != null) {
                            e.addSuppressed(failure);
                        }
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (failure == null) {
                            logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
                            listener.onResponse(result);
                        } else {
                            logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                            listener.onFailure(failure);
                        }
                    }
                });
            }
        });
    }, listener::onFailure);
}

Example 2 with RepositoryCleanupResult

use of org.opensearch.repositories.RepositoryCleanupResult in project OpenSearch by opensearch-project.

the class BlobStoreRepository method cleanup.

/**
 * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the
 * repository.
 * TODO: Add shard level cleanups
 * TODO: Add unreferenced index metadata cleanup
 * <ul>
 *     <li>Deleting stale indices {@link #cleanupStaleIndices}</li>
 *     <li>Deleting unreferenced root level blobs {@link #cleanupStaleRootFiles}</li>
 * </ul>
 * @param repositoryStateId     Current repository state id
 * @param repositoryMetaVersion version of the updated repository metadata to write
 * @param listener              Listener to complete when done
 */
public void cleanup(long repositoryStateId, Version repositoryMetaVersion, ActionListener<RepositoryCleanupResult> listener) {
    try {
        if (isReadOnly()) {
            throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository");
        }
        Map<String, BlobMetadata> rootBlobs = blobContainer().listBlobs();
        final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs);
        final Map<String, BlobContainer> foundIndices = blobStore().blobContainer(indicesPath()).children();
        final Set<String> survivingIndexIds = repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet());
        final List<String> staleRootBlobs = staleRootBlobs(repositoryData, rootBlobs.keySet());
        if (survivingIndexIds.equals(foundIndices.keySet()) && staleRootBlobs.isEmpty()) {
            // Nothing to clean up we return
            listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO));
        } else {
            // write new index-N blob to ensure concurrent operations will fail
            writeIndexGen(repositoryData, repositoryStateId, repositoryMetaVersion, Function.identity(), ActionListener.wrap(v -> cleanupStaleBlobs(Collections.emptyList(), foundIndices, rootBlobs, repositoryData, ActionListener.map(listener, RepositoryCleanupResult::new)), listener::onFailure));
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}