Examples with DeleteResult org.opensearch.common.blobstore.DeleteResult used on opensource projects

Example 1 with DeleteResult

use of org.opensearch.common.blobstore.DeleteResult in project OpenSearch by opensearch-project.

the class FsBlobContainer method delete.

@Override
public DeleteResult delete() throws IOException {
    final AtomicLong filesDeleted = new AtomicLong(0L);
    final AtomicLong bytesDeleted = new AtomicLong(0L);
    Files.walkFileTree(path, new SimpleFileVisitor<Path>() {

        @Override
        public FileVisitResult postVisitDirectory(Path dir, IOException impossible) throws IOException {
            assert impossible == null;
            Files.delete(dir);
            return FileVisitResult.CONTINUE;
        }

        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
            Files.delete(file);
            filesDeleted.incrementAndGet();
            bytesDeleted.addAndGet(attrs.size());
            return FileVisitResult.CONTINUE;
        }
    });
    return new DeleteResult(filesDeleted.get(), bytesDeleted.get());
}

Example 2 with DeleteResult

use of org.opensearch.common.blobstore.DeleteResult in project OpenSearch by opensearch-project.

the class AzureBlobStore method deleteBlobDirectory.

public DeleteResult deleteBlobDirectory(String path, Executor executor) throws URISyntaxException, BlobStorageException, IOException {
    final Tuple<BlobServiceClient, Supplier<Context>> client = client();
    final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
    final Collection<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
    final AtomicLong outstanding = new AtomicLong(1L);
    final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
    final AtomicLong blobsDeleted = new AtomicLong();
    final AtomicLong bytesDeleted = new AtomicLong();
    final ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setPrefix(path);
    SocketAccess.doPrivilegedVoidException(() -> {
        String continuationToken = null;
        do {
            // Fetch one page at a time, others are going to be fetched by continuation token
            // TODO: reconsider reverting to simplified approach once https://github.com/Azure/azure-sdk-for-java/issues/26064
            // gets addressed.
            final Optional<PagedResponse<BlobItem>> pageOpt = blobContainer.listBlobs(listBlobsOptions, timeout()).streamByPage(continuationToken).findFirst();
            if (!pageOpt.isPresent()) {
                // No more pages, should never happen
                break;
            }
            final PagedResponse<BlobItem> page = pageOpt.get();
            for (final BlobItem blobItem : page.getValue()) {
                // Skipping prefixes as those are not deletable and should not be there
                assert (blobItem.isPrefix() == null || !blobItem.isPrefix()) : "Only blobs (not prefixes) are expected";
                outstanding.incrementAndGet();
                executor.execute(new AbstractRunnable() {

                    @Override
                    protected void doRun() throws Exception {
                        final long len = blobItem.getProperties().getContentLength();
                        final BlobClient azureBlob = blobContainer.getBlobClient(blobItem.getName());
                        logger.trace(() -> new ParameterizedMessage("container [{}]: blob [{}] found. removing.", container, blobItem.getName()));
                        final Response<Void> response = azureBlob.deleteWithResponse(null, null, timeout(), client.v2().get());
                        logger.trace(() -> new ParameterizedMessage("container [{}]: blob [{}] deleted status [{}].", container, blobItem.getName(), response.getStatusCode()));
                        blobsDeleted.incrementAndGet();
                        if (len >= 0) {
                            bytesDeleted.addAndGet(len);
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        exceptions.add(e);
                    }

                    @Override
                    public void onAfter() {
                        if (outstanding.decrementAndGet() == 0) {
                            result.onResponse(null);
                        }
                    }
                });
            }
            // Fetch next continuation token
            continuationToken = page.getContinuationToken();
        } while (StringUtils.isNotBlank(continuationToken));
    });
    if (outstanding.decrementAndGet() == 0) {
        result.onResponse(null);
    }
    result.actionGet();
    if (exceptions.isEmpty() == false) {
        final IOException ex = new IOException("Deleting directory [" + path + "] failed");
        exceptions.forEach(ex::addSuppressed);
        throw ex;
    }
    return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
}

Example 3 with DeleteResult

use of org.opensearch.common.blobstore.DeleteResult in project OpenSearch by opensearch-project.

the class GoogleCloudStorageBlobStore method deleteDirectory.

/**
 * Deletes the given path and all its children.
 *
 * @param pathStr Name of path to delete
 */
DeleteResult deleteDirectory(String pathStr) throws IOException {
    return SocketAccess.doPrivilegedIOException(() -> {
        DeleteResult deleteResult = DeleteResult.ZERO;
        Page<Blob> page = client().list(bucketName, BlobListOption.prefix(pathStr));
        do {
            final Collection<String> blobsToDelete = new ArrayList<>();
            final AtomicLong blobsDeleted = new AtomicLong(0L);
            final AtomicLong bytesDeleted = new AtomicLong(0L);
            page.getValues().forEach(b -> {
                blobsToDelete.add(b.getName());
                blobsDeleted.incrementAndGet();
                bytesDeleted.addAndGet(b.getSize());
            });
            deleteBlobsIgnoringIfNotExists(blobsToDelete);
            deleteResult = deleteResult.add(blobsDeleted.get(), bytesDeleted.get());
            page = page.getNextPage();
        } while (page != null);
        return deleteResult;
    });
}

Example 4 with DeleteResult

use of org.opensearch.common.blobstore.DeleteResult in project OpenSearch by opensearch-project.

the class BlobStoreRepository method executeOneStaleIndexDelete.

private void executeOneStaleIndexDelete(BlockingQueue<Map.Entry<String, BlobContainer>> staleIndicesToDelete, GroupedActionListener<DeleteResult> listener) throws InterruptedException {
    Map.Entry<String, BlobContainer> indexEntry = staleIndicesToDelete.poll(0L, TimeUnit.MILLISECONDS);
    if (indexEntry != null) {
        final String indexSnId = indexEntry.getKey();
        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(listener, () -> {
            DeleteResult deleteResult = DeleteResult.ZERO;
            try {
                logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId);
                deleteResult = indexEntry.getValue().delete();
                logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId);
            } catch (IOException e) {
                logger.warn(() -> new ParameterizedMessage("[{}] index {} is no longer part of any snapshots in the repository, " + "but failed to clean up their index folders", metadata.name(), indexSnId), e);
            } catch (Exception e) {
                assert false : e;
                logger.warn(new ParameterizedMessage("[{}] Exception during single stale index delete", metadata.name()), e);
            }
            executeOneStaleIndexDelete(staleIndicesToDelete, listener);
            return deleteResult;
        }));
    }
}

Example 5 with DeleteResult

use of org.opensearch.common.blobstore.DeleteResult in project OpenSearch by opensearch-project.

the class BlobStoreRepository method cleanupStaleBlobs.

/**
 * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing
 * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository and with
 * parameters {@code foundIndices}, {@code rootBlobs}
 *
 * @param deletedSnapshots if this method is called as part of a delete operation, the snapshot ids just deleted or empty if called as
 *                         part of a repository cleanup
 * @param foundIndices     all indices blob containers found in the repository before {@code newRepoData} was written
 * @param rootBlobs        all blobs found directly under the repository root
 * @param newRepoData      new repository data that was just written
 * @param listener         listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation
 */
private void cleanupStaleBlobs(Collection<SnapshotId> deletedSnapshots, Map<String, BlobContainer> foundIndices, Map<String, BlobMetadata> rootBlobs, RepositoryData newRepoData, ActionListener<DeleteResult> listener) {
    final GroupedActionListener<DeleteResult> groupedListener = new GroupedActionListener<>(ActionListener.wrap(deleteResults -> {
        DeleteResult deleteResult = DeleteResult.ZERO;
        for (DeleteResult result : deleteResults) {
            deleteResult = deleteResult.add(result);
        }
        listener.onResponse(deleteResult);
    }, listener::onFailure), 2);
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final List<String> staleRootBlobs = staleRootBlobs(newRepoData, rootBlobs.keySet());
    if (staleRootBlobs.isEmpty()) {
        groupedListener.onResponse(DeleteResult.ZERO);
    } else {
        executor.execute(ActionRunnable.supply(groupedListener, () -> {
            List<String> deletedBlobs = cleanupStaleRootFiles(newRepoData.getGenId() - 1, deletedSnapshots, staleRootBlobs);
            return new DeleteResult(deletedBlobs.size(), deletedBlobs.stream().mapToLong(name -> rootBlobs.get(name).length()).sum());
        }));
    }
    final Set<String> survivingIndexIds = newRepoData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet());
    if (foundIndices.keySet().equals(survivingIndexIds)) {
        groupedListener.onResponse(DeleteResult.ZERO);
    } else {
        cleanupStaleIndices(foundIndices, survivingIndexIds, groupedListener);
    }
}