Examples with PlainBlobMetadata org.opensearch.common.blobstore.support.PlainBlobMetadata used on opensource projects

Example 1 with PlainBlobMetadata

use of org.opensearch.common.blobstore.support.PlainBlobMetadata in project OpenSearch by opensearch-project.

the class HdfsBlobContainer method listBlobsByPrefix.

@Override
public Map<String, BlobMetadata> listBlobsByPrefix(@Nullable final String prefix) throws IOException {
    FileStatus[] files = store.execute(fileContext -> fileContext.util().listStatus(path, path -> prefix == null || path.getName().startsWith(prefix)));
    Map<String, BlobMetadata> map = new LinkedHashMap<>();
    for (FileStatus file : files) {
        if (file.isFile()) {
            map.put(file.getPath().getName(), new PlainBlobMetadata(file.getPath().getName(), file.getLen()));
        }
    }
    return Collections.unmodifiableMap(map);
}

Example 2 with PlainBlobMetadata

use of org.opensearch.common.blobstore.support.PlainBlobMetadata in project OpenSearch by opensearch-project.

the class AzureBlobStore method listBlobsByPrefix.

public Map<String, BlobMetadata> listBlobsByPrefix(String keyPath, String prefix) throws URISyntaxException, BlobStorageException {
    final Map<String, BlobMetadata> blobsBuilder = new HashMap<String, BlobMetadata>();
    final Tuple<BlobServiceClient, Supplier<Context>> client = client();
    final BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
    logger.trace(() -> new ParameterizedMessage("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix));
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
    final ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(new BlobListDetails().setRetrieveMetadata(true)).setPrefix(keyPath + (prefix == null ? "" : prefix));
    SocketAccess.doPrivilegedVoidException(() -> {
        String continuationToken = null;
        do {
            // Fetch one page at a time, others are going to be fetched by continuation token
            // TODO: reconsider reverting to simplified approach once https://github.com/Azure/azure-sdk-for-java/issues/26064
            // gets addressed
            final Optional<PagedResponse<BlobItem>> pageOpt = blobContainer.listBlobsByHierarchy("/", listBlobsOptions, timeout()).streamByPage(continuationToken).findFirst();
            if (!pageOpt.isPresent()) {
                // No more pages, should never happen
                break;
            }
            final PagedResponse<BlobItem> page = pageOpt.get();
            for (final BlobItem blobItem : page.getValue()) {
                // Skipping over the prefixes, only look for the blobs
                if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
                    continue;
                }
                final String name = getBlobName(blobItem.getName(), container, keyPath);
                logger.trace(() -> new ParameterizedMessage("blob name [{}]", name));
                final BlobItemProperties properties = blobItem.getProperties();
                logger.trace(() -> new ParameterizedMessage("blob name [{}], size [{}]", name, properties.getContentLength()));
                blobsBuilder.put(name, new PlainBlobMetadata(name, properties.getContentLength()));
            }
            // Fetch next continuation token
            continuationToken = page.getContinuationToken();
        } while (StringUtils.isNotBlank(continuationToken));
    });
    return MapBuilder.newMapBuilder(blobsBuilder).immutableMap();
}

Example 3 with PlainBlobMetadata

use of org.opensearch.common.blobstore.support.PlainBlobMetadata in project OpenSearch by opensearch-project.

the class FsBlobContainer method listBlobsByPrefix.

@Override
public Map<String, BlobMetadata> listBlobsByPrefix(String blobNamePrefix) throws IOException {
    Map<String, BlobMetadata> builder = new HashMap<>();
    blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, blobNamePrefix + "*")) {
        for (Path file : stream) {
            final BasicFileAttributes attrs;
            try {
                attrs = Files.readAttributes(file, BasicFileAttributes.class);
            } catch (FileNotFoundException | NoSuchFileException e) {
                // The file was concurrently deleted between listing files and trying to get its attributes so we skip it here
                continue;
            }
            if (attrs.isRegularFile()) {
                builder.put(file.getFileName().toString(), new PlainBlobMetadata(file.getFileName().toString(), attrs.size()));
            }
        }
    }
    return unmodifiableMap(builder);
}

Example 4 with PlainBlobMetadata

use of org.opensearch.common.blobstore.support.PlainBlobMetadata in project OpenSearch by opensearch-project.

the class AbstractThirdPartyRepositoryTestCase method testListChildren.

public void testListChildren() throws Exception {
    final BlobStoreRepository repo = getRepository();
    final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
    final Executor genericExec = repo.threadPool().generic();
    final int testBlobLen = randomIntBetween(1, 100);
    genericExec.execute(ActionRunnable.run(future, () -> {
        final BlobStore blobStore = repo.blobStore();
        blobStore.blobContainer(repo.basePath().add("foo")).writeBlob("nested-blob", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
        blobStore.blobContainer(repo.basePath().add("foo").add("nested")).writeBlob("bar", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
        blobStore.blobContainer(repo.basePath().add("foo").add("nested2")).writeBlob("blub", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
    }));
    future.actionGet();
    assertChildren(repo.basePath(), Collections.singleton("foo"));
    assertBlobsByPrefix(repo.basePath(), "fo", Collections.emptyMap());
    assertChildren(repo.basePath().add("foo"), Arrays.asList("nested", "nested2"));
    assertBlobsByPrefix(repo.basePath().add("foo"), "nest", Collections.singletonMap("nested-blob", new PlainBlobMetadata("nested-blob", testBlobLen)));
    assertChildren(repo.basePath().add("foo").add("nested"), Collections.emptyList());
    if (randomBoolean()) {
        deleteAndAssertEmpty(repo.basePath());
    } else {
        deleteAndAssertEmpty(repo.basePath().add("foo"));
    }
}