Examples with BlobContainerClient com.azure.storage.blob.BlobContainerClient used on opensource projects

Example 1 with BlobContainerClient

use of com.azure.storage.blob.BlobContainerClient in project beam by apache.

the class AzureBlobStoreFileSystem method expand.

/**
 * Expands a pattern into {@link MatchResult}.
 */
@VisibleForTesting
MatchResult expand(AzfsResourceId azfsPattern) {
    checkArgument(azfsPattern.isWildcard(), "The resource id should be a wildcard.");
    String blobPrefix = azfsPattern.getBlobNonWildcardPrefix();
    Pattern wildcardAsRegexp = Pattern.compile(wildcardToRegexp(azfsPattern.getBlob()));
    LOG.debug("matching files in container {}, prefix {} against pattern {}", azfsPattern.getContainer(), blobPrefix, wildcardAsRegexp.toString());
    ListBlobsOptions listOptions = new ListBlobsOptions().setPrefix(blobPrefix);
    Duration timeout = Duration.ofMinutes(1);
    String account = azfsPattern.getAccount();
    String container = azfsPattern.getContainer();
    BlobContainerClient blobContainerClient = client.get().getBlobContainerClient(container);
    PagedIterable<BlobItem> blobs = blobContainerClient.listBlobs(listOptions, timeout);
    List<MatchResult.Metadata> results = new ArrayList<>();
    blobs.forEach(blob -> {
        String name = blob.getName();
        if (wildcardAsRegexp.matcher(name).matches() && !name.endsWith("/")) {
            LOG.debug("Matched object: azfs://{}/{}/{}", account, container, name);
            BlobProperties properties = blobContainerClient.getBlobClient(name).getProperties();
            AzfsResourceId rid = AzfsResourceId.fromComponents(account, container, name).withSize(properties.getBlobSize()).withLastModified(Date.from(properties.getLastModified().toInstant()));
            results.add(toMetadata(rid, properties.getContentEncoding(), properties.getETag()));
        }
    });
    return MatchResult.create(MatchResult.Status.OK, results);
}

Example 2 with BlobContainerClient

use of com.azure.storage.blob.BlobContainerClient in project beam by apache.

the class AzureBlobStoreFileSystem method copy.

@VisibleForTesting
void copy(AzfsResourceId sourcePath, AzfsResourceId destinationPath) throws IOException {
    checkArgument(sourcePath.getBlob() != null && destinationPath.getBlob() != null, "This method is intended to copy file-like resources, not directories.");
    // get source blob client
    BlobClient srcBlobClient = client.get().getBlobContainerClient(sourcePath.getContainer()).getBlobClient(sourcePath.getBlob());
    if (!srcBlobClient.exists()) {
        throw new FileNotFoundException("The copy source does not exist.");
    }
    // get destination blob client
    BlobContainerClient destBlobContainerClient = client.get().getBlobContainerClient(destinationPath.getContainer());
    if (!destBlobContainerClient.exists()) {
        client.get().createBlobContainer(destinationPath.getContainer());
        LOG.info("Created a container called {}", destinationPath.getContainer());
    }
    BlobClient destBlobClient = destBlobContainerClient.getBlobClient(destinationPath.getBlob());
    destBlobClient.copyFromUrl(srcBlobClient.getBlobUrl() + generateSasToken());
}

Example 3 with BlobContainerClient

use of com.azure.storage.blob.BlobContainerClient in project beam by apache.

the class AzureBlobStoreFileSystem method open.

@Override
protected ReadableByteChannel open(AzfsResourceId resourceId) throws IOException {
    BlobContainerClient containerClient = client.get().getBlobContainerClient(resourceId.getContainer());
    if (!containerClient.exists()) {
        throw new FileNotFoundException("The requested file doesn't exist.");
    }
    BlobClient blobClient = containerClient.getBlobClient(resourceId.getBlob());
    if (!blobClient.exists()) {
        throw new FileNotFoundException("The requested file doesn't exist.");
    }
    LOG.info("Creating a ReadableByteChannel for {}", resourceId);
    return new AzureReadableSeekableByteChannel(blobClient);
}

Example 4 with BlobContainerClient

use of com.azure.storage.blob.BlobContainerClient in project beam by apache.

the class AzureBlobStoreFileSystemTest method testGlobExpansion.

@Test
@Ignore
public void testGlobExpansion() throws IOException {
    // TODO: Write this test with mocks - see GcsFileSystemTest
    String container = "test-container" + randomUUID();
    BlobContainerClient blobContainerClient = azureBlobStoreFileSystem.getClient().createBlobContainer(container);
    // Create files
    List<String> blobNames = new ArrayList<>();
    blobNames.add("testdirectory/file1name");
    blobNames.add("testdirectory/file2name");
    blobNames.add("testdirectory/file3name");
    blobNames.add("testdirectory/otherfile");
    blobNames.add("testotherdirectory/file4name");
    for (String blob : blobNames) {
        blobContainerClient.getBlobClient(blob).uploadFromFile("src/test/resources/in.txt");
    }
    // Test patterns
    {
        AzfsResourceId pattern = AzfsResourceId.fromUri("azfs://account/" + container + "/testdirectory/file*");
        List<String> expectedFiles = ImmutableList.of("azfs://account/" + container + "/testdirectory/file1name", "azfs://account/" + container + "/testdirectory/file2name", "azfs://account/" + container + "/testdirectory/file3name");
        assertThat(expectedFiles, contains(toFilenames(azureBlobStoreFileSystem.expand(pattern)).toArray()));
    }
    {
        AzfsResourceId pattern = AzfsResourceId.fromUri("azfs://account/" + container + "/testdirectory/file[1-3]*");
        List<String> expectedFiles = ImmutableList.of("azfs://account/" + container + "/testdirectory/file1name", "azfs://account/" + container + "/testdirectory/file2name", "azfs://account/" + container + "/testdirectory/file3name");
        assertThat(expectedFiles, contains(toFilenames(azureBlobStoreFileSystem.expand(pattern)).toArray()));
    }
    {
        AzfsResourceId pattern = AzfsResourceId.fromUri("azfs://account/" + container + "/testdirectory/file?name");
        List<String> expectedFiles = ImmutableList.of("azfs://account/" + container + "/testdirectory/file1name", "azfs://account/" + container + "/testdirectory/file2name", "azfs://account/" + container + "/testdirectory/file3name");
        assertThat(expectedFiles, contains(toFilenames(azureBlobStoreFileSystem.expand(pattern)).toArray()));
    }
    {
        AzfsResourceId pattern = AzfsResourceId.fromUri("azfs://account/" + container + "/test*ectory/fi*name");
        List<String> expectedFiles = ImmutableList.of("azfs://account/" + container + "/testdirectory/file1name", "azfs://account/" + container + "/testdirectory/file2name", "azfs://account/" + container + "/testdirectory/file3name", "azfs://account/" + container + "/testotherdirectory/file4name");
        assertThat(expectedFiles, contains(toFilenames(azureBlobStoreFileSystem.expand(pattern)).toArray()));
    }
    // Clean up
    blobContainerClient.delete();
}

Example 5 with BlobContainerClient

use of com.azure.storage.blob.BlobContainerClient in project beam by apache.

the class AzureBlobStoreFileSystemTest method testMatch.

@Test
@Ignore
public void testMatch() throws Exception {
    // TODO: Write this test with mocks - see GcsFileSystemTest
    String container = "test-container" + randomUUID();
    BlobContainerClient blobContainerClient = azureBlobStoreFileSystem.getClient().createBlobContainer(container);
    // Create files
    List<String> blobNames = new ArrayList<>();
    blobNames.add("testdirectory/file1name");
    blobNames.add("testdirectory/file2name");
    blobNames.add("testdirectory/file3name");
    blobNames.add("testdirectory/file4name");
    blobNames.add("testdirectory/otherfile");
    blobNames.add("testotherdirectory/anotherfile");
    for (String blob : blobNames) {
        blobContainerClient.getBlobClient(blob).uploadFromFile("src/test/resources/in.txt");
    }
    List<String> specs = ImmutableList.of("azfs://account/" + container + "/testdirectory/file[1-3]*", "azfs://account/" + container + "/testdirectory/non-exist-file", "azfs://account/" + container + "/testdirectory/otherfile");
    List<MatchResult> matchResults = azureBlobStoreFileSystem.match(specs);
    // Confirm that match results are as expected
    assertEquals(3, matchResults.size());
    assertEquals(MatchResult.Status.OK, matchResults.get(0).status());
    assertThat(ImmutableList.of("azfs://account/" + container + "/testdirectory/file1name", "azfs://account/" + container + "/testdirectory/file2name", "azfs://account/" + container + "/testdirectory/file3name"), contains(toFilenames(matchResults.get(0)).toArray()));
    assertEquals(MatchResult.Status.NOT_FOUND, matchResults.get(1).status());
    assertEquals(MatchResult.Status.OK, matchResults.get(2).status());
    assertThat(ImmutableList.of("azfs://account/" + container + "/testdirectory/otherfile"), contains(toFilenames(matchResults.get(2)).toArray()));
    blobContainerClient.delete();
}