Examples with BlobMetadata org.opensearch.common.blobstore.BlobMetadata used on opensource projects

Example 6 with BlobMetadata

use of org.opensearch.common.blobstore.BlobMetadata in project OpenSearch by opensearch-project.

the class BlobStoreRepository method cleanup.

/**
 * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the
 * repository.
 * TODO: Add shard level cleanups
 * TODO: Add unreferenced index metadata cleanup
 * <ul>
 *     <li>Deleting stale indices {@link #cleanupStaleIndices}</li>
 *     <li>Deleting unreferenced root level blobs {@link #cleanupStaleRootFiles}</li>
 * </ul>
 * @param repositoryStateId     Current repository state id
 * @param repositoryMetaVersion version of the updated repository metadata to write
 * @param listener              Listener to complete when done
 */
public void cleanup(long repositoryStateId, Version repositoryMetaVersion, ActionListener<RepositoryCleanupResult> listener) {
    try {
        if (isReadOnly()) {
            throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository");
        }
        Map<String, BlobMetadata> rootBlobs = blobContainer().listBlobs();
        final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs);
        final Map<String, BlobContainer> foundIndices = blobStore().blobContainer(indicesPath()).children();
        final Set<String> survivingIndexIds = repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet());
        final List<String> staleRootBlobs = staleRootBlobs(repositoryData, rootBlobs.keySet());
        if (survivingIndexIds.equals(foundIndices.keySet()) && staleRootBlobs.isEmpty()) {
            // Nothing to clean up we return
            listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO));
        } else {
            // write new index-N blob to ensure concurrent operations will fail
            writeIndexGen(repositoryData, repositoryStateId, repositoryMetaVersion, Function.identity(), ActionListener.wrap(v -> cleanupStaleBlobs(Collections.emptyList(), foundIndices, rootBlobs, repositoryData, ActionListener.map(listener, RepositoryCleanupResult::new)), listener::onFailure));
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}

Example 7 with BlobMetadata

use of org.opensearch.common.blobstore.BlobMetadata in project OpenSearch by opensearch-project.

the class BlobStoreRepository method doDeleteShardSnapshots.

/**
 * After updating the {@link RepositoryData} each of the shards directories is individually first moved to the next shard generation
 * and then has all now unreferenced blobs in it deleted.
 *
 * @param snapshotIds       SnapshotIds to delete
 * @param repositoryStateId Expected repository state id
 * @param foundIndices      All indices folders found in the repository before executing any writes to the repository during this
 *                          delete operation
 * @param rootBlobs         All blobs found at the root of the repository before executing any writes to the repository during this
 *                          delete operation
 * @param repositoryData    RepositoryData found the in the repository before executing this delete
 * @param listener          Listener to invoke once finished
 */
private void doDeleteShardSnapshots(Collection<SnapshotId> snapshotIds, long repositoryStateId, Map<String, BlobContainer> foundIndices, Map<String, BlobMetadata> rootBlobs, RepositoryData repositoryData, Version repoMetaVersion, ActionListener<RepositoryData> listener) {
    if (SnapshotsService.useShardGenerations(repoMetaVersion)) {
        // First write the new shard state metadata (with the removed snapshot) and compute deletion targets
        final StepListener<Collection<ShardSnapshotMetaDeleteResult>> writeShardMetaDataAndComputeDeletesStep = new StepListener<>();
        writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, true, writeShardMetaDataAndComputeDeletesStep);
        // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows:
        // 1. Remove the snapshots from the list of existing snapshots
        // 2. Update the index shard generations of all updated shard folders
        // 
        // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created
        // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only
        // written if all shard paths have been successfully updated.
        final StepListener<RepositoryData> writeUpdatedRepoDataStep = new StepListener<>();
        writeShardMetaDataAndComputeDeletesStep.whenComplete(deleteResults -> {
            final ShardGenerations.Builder builder = ShardGenerations.builder();
            for (ShardSnapshotMetaDeleteResult newGen : deleteResults) {
                builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration);
            }
            final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, builder.build());
            writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure));
        }, listener::onFailure);
        // Once we have updated the repository, run the clean-ups
        writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> {
            // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
            final ActionListener<Void> afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(updatedRepoData)), 2);
            cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener);
            asyncCleanupUnlinkedShardLevelBlobs(repositoryData, snapshotIds, writeShardMetaDataAndComputeDeletesStep.result(), afterCleanupsListener);
        }, listener::onFailure);
    } else {
        // Write the new repository data first (with the removed snapshot), using no shard generations
        final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY);
        writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), ActionListener.wrap(newRepoData -> {
            // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
            final ActionListener<Void> afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(newRepoData)), 2);
            cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, newRepoData, afterCleanupsListener);
            final StepListener<Collection<ShardSnapshotMetaDeleteResult>> writeMetaAndComputeDeletesStep = new StepListener<>();
            writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, false, writeMetaAndComputeDeletesStep);
            writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> asyncCleanupUnlinkedShardLevelBlobs(repositoryData, snapshotIds, deleteResults, afterCleanupsListener), afterCleanupsListener::onFailure);
        }, listener::onFailure));
    }
}

Example 8 with BlobMetadata

use of org.opensearch.common.blobstore.BlobMetadata in project OpenSearch by opensearch-project.

the class AzureBlobStore method listBlobsByPrefix.

public Map<String, BlobMetadata> listBlobsByPrefix(String keyPath, String prefix) throws URISyntaxException, BlobStorageException {
    final Map<String, BlobMetadata> blobsBuilder = new HashMap<String, BlobMetadata>();
    final Tuple<BlobServiceClient, Supplier<Context>> client = client();
    final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
    logger.trace(() -> new ParameterizedMessage("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix));
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
    final ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(new BlobListDetails().setRetrieveMetadata(true)).setPrefix(keyPath + (prefix == null ? "" : prefix));
    SocketAccess.doPrivilegedVoidException(() -> {
        for (final BlobItem blobItem : blobContainer.listBlobsByHierarchy("/", listBlobsOptions, timeout())) {
            // Skipping over the prefixes, only look for the blobs
            if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
                continue;
            }
            final String name = getBlobName(blobItem.getName(), container, keyPath);
            logger.trace(() -> new ParameterizedMessage("blob name [{}]", name));
            final BlobItemProperties properties = blobItem.getProperties();
            logger.trace(() -> new ParameterizedMessage("blob name [{}], size [{}]", name, properties.getContentLength()));
            blobsBuilder.put(name, new PlainBlobMetadata(name, properties.getContentLength()));
        }
    });
    return MapBuilder.newMapBuilder(blobsBuilder).immutableMap();
}

Example 9 with BlobMetadata

use of org.opensearch.common.blobstore.BlobMetadata in project OpenSearch by opensearch-project.

the class FsBlobContainer method listBlobsByPrefix.

@Override
public Map<String, BlobMetadata> listBlobsByPrefix(String blobNamePrefix) throws IOException {
    Map<String, BlobMetadata> builder = new HashMap<>();
    blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, blobNamePrefix + "*")) {
        for (Path file : stream) {
            final BasicFileAttributes attrs;
            try {
                attrs = Files.readAttributes(file, BasicFileAttributes.class);
            } catch (FileNotFoundException | NoSuchFileException e) {
                // The file was concurrently deleted between listing files and trying to get its attributes so we skip it here
                continue;
            }
            if (attrs.isRegularFile()) {
                builder.put(file.getFileName().toString(), new PlainBlobMetadata(file.getFileName().toString(), attrs.size()));
            }
        }
    }
    return unmodifiableMap(builder);
}