Examples with CloudBlob com.microsoft.azure.storage.blob.CloudBlob used on opensource projects

Example 11 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.

the class ITFetchAzureBlobStorage method testFetchingBlob.

@Test
public void testFetchingBlob() throws InvalidKeyException, URISyntaxException, StorageException, IOException {
    String containerName = String.format("%s-%s", AzureTestUtil.TEST_CONTAINER_NAME_PREFIX, UUID.randomUUID());
    CloudBlobContainer container = AzureTestUtil.getContainer(containerName);
    container.createIfNotExists();
    CloudBlob blob = container.getBlockBlobReference(AzureTestUtil.TEST_BLOB_NAME);
    byte[] buf = "0123456789".getBytes();
    InputStream in = new ByteArrayInputStream(buf);
    blob.upload(in, 10);
    final TestRunner runner = TestRunners.newTestRunner(new FetchAzureBlobStorage());
    try {
        runner.setValidateExpressionUsage(true);
        runner.setProperty(AzureStorageUtils.ACCOUNT_NAME, AzureTestUtil.getAccountName());
        runner.setProperty(AzureStorageUtils.ACCOUNT_KEY, AzureTestUtil.getAccountKey());
        runner.setProperty(AzureStorageUtils.CONTAINER, containerName);
        runner.setProperty(FetchAzureBlobStorage.BLOB, "${azure.blobname}");
        final Map<String, String> attributes = new HashMap<>();
        attributes.put("azure.primaryUri", "https://" + AzureTestUtil.getAccountName() + ".blob.core.windows.net/" + containerName + "/" + AzureTestUtil.TEST_BLOB_NAME);
        attributes.put("azure.blobname", AzureTestUtil.TEST_BLOB_NAME);
        attributes.put("azure.blobtype", AzureStorageUtils.BLOCK);
        runner.enqueue(new byte[0], attributes);
        runner.run();
        runner.assertAllFlowFilesTransferred(AbstractAzureBlobProcessor.REL_SUCCESS, 1);
        List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(FetchAzureBlobStorage.REL_SUCCESS);
        for (MockFlowFile flowFile : flowFilesForRelationship) {
            flowFile.assertContentEquals("0123456789".getBytes());
            flowFile.assertAttributeEquals("azure.length", "10");
        }
    } finally {
        container.deleteIfExists();
    }
}

Example 12 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project wildfly-camel by wildfly-extras.

the class AzureIntegrationTest method testAppendBlob.

@Test
public void testAppendBlob() throws Exception {
    StorageCredentials creds = getStorageCredentials("camelblob", System.getenv(AZURE_STORAGE_BLOB));
    Assume.assumeNotNull("Credentials not null", creds);
    CamelContext camelctx = createCamelContext(creds);
    camelctx.addRoutes(new RouteBuilder() {

        public void configure() throws Exception {
            from("direct:start").to("azure-blob://camelblob/container1/blobAppend?credentials=#creds&operation=updateAppendBlob");
            from("azure-blob://camelblob/container1/blobAppend?credentials=#creds&blobType=appendblob").to("mock:read");
            from("direct:list").to("azure-blob://camelblob/container1?credentials=#creds&operation=listBlobs");
        }
    });
    camelctx.start();
    try {
        MockEndpoint mockRead = camelctx.getEndpoint("mock:read", MockEndpoint.class);
        mockRead.expectedBodiesReceived("Append Blob");
        mockRead.expectedMessageCount(1);
        ProducerTemplate producer = camelctx.createProducerTemplate();
        Iterator<?> it = producer.requestBody("direct:list", null, Iterable.class).iterator();
        Assert.assertFalse("No Blob exists", it.hasNext());
        // append to blob
        producer.sendBody("direct:start", "Append Blob");
        mockRead.assertIsSatisfied();
        it = producer.requestBody("direct:list", null, Iterable.class).iterator();
        Assert.assertTrue("Blob exists", it.hasNext());
        CloudBlob blob = (CloudAppendBlob) it.next();
        blob.delete();
        it = producer.requestBody("direct:list", null, Iterable.class).iterator();
        Assert.assertFalse("No Blob exists", it.hasNext());
    } finally {
        camelctx.stop();
    }
}

Example 13 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project jackrabbit-oak by apache.

the class AzureArchiveManager method recoverEntries.

@Override
public void recoverEntries(String archiveName, LinkedHashMap<UUID, byte[]> entries) throws IOException {
    Pattern pattern = Pattern.compile(AzureUtilities.SEGMENT_FILE_NAME_PATTERN);
    List<RecoveredEntry> entryList = new ArrayList<>();
    for (CloudBlob b : getBlobList(archiveName)) {
        String name = getName(b);
        Matcher m = pattern.matcher(name);
        if (!m.matches()) {
            continue;
        }
        int position = Integer.parseInt(m.group(1), 16);
        UUID uuid = UUID.fromString(m.group(2));
        long length = b.getProperties().getLength();
        if (length > 0) {
            byte[] data = new byte[(int) length];
            try {
                b.downloadToByteArray(data, 0);
            } catch (StorageException e) {
                throw new IOException(e);
            }
            entryList.add(new RecoveredEntry(position, uuid, data, name));
        }
    }
    Collections.sort(entryList);
    int i = 0;
    for (RecoveredEntry e : entryList) {
        if (e.position != i) {
            log.warn("Missing entry {}.??? when recovering {}. No more segments will be read.", String.format("%04X", i), archiveName);
            break;
        }
        log.info("Recovering segment {}/{}", archiveName, e.fileName);
        entries.put(e.uuid, e.data);
        i++;
    }
}

Example 14 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project components by Talend.

the class AzureStorageListReader method start.

@Override
public boolean start() throws IOException {
    String mycontainer = properties.container.getValue();
    List<CloudBlob> blobs = new ArrayList<>();
    // build a list with remote blobs to fetch
    List<RemoteBlob> remoteBlobs = ((AzureStorageSource) getCurrentSource()).getRemoteBlobs();
    try {
        for (RemoteBlob rmtb : remoteBlobs) {
            for (ListBlobItem blob : azureStorageBlobService.listBlobs(mycontainer, rmtb.prefix, rmtb.include)) {
                if (blob instanceof CloudBlob) {
                    blobs.add((CloudBlob) blob);
                }
            }
        }
        startable = !blobs.isEmpty();
        blobsIterator = blobs.iterator();
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        LOGGER.error(e.getLocalizedMessage());
        if (properties.dieOnError.getValue()) {
            throw new ComponentException(e);
        }
    }
    if (startable) {
        dataCount++;
        currentBlob = blobsIterator.next();
        IndexedRecord dataRecord = new GenericData.Record(properties.schema.schema.getValue());
        dataRecord.put(0, currentBlob.getName());
        Schema rootSchema = RootSchemaUtils.createRootSchema(properties.schema.schema.getValue(), properties.outOfBandSchema);
        currentRecord = new GenericData.Record(rootSchema);
        currentRecord.put(0, dataRecord);
        currentRecord.put(1, dataRecord);
    }
    return startable;
}

Example 15 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method createDiskStateObject.

private DiskState createDiskStateObject(StorageEnumContext context, ListBlobItem blob, String containerLink, DiskState oldDiskState) {
    DiskState diskState = new DiskState();
    diskState.name = getAzureBlobName(blob);
    if (containerLink != null) {
        diskState.storageDescriptionLink = containerLink;
    }
    diskState.resourcePoolLink = context.request.resourcePoolLink;
    diskState.computeHostLink = context.parentCompute.documentSelfLink;
    diskState.endpointLink = context.request.endpointLink;
    if (oldDiskState != null) {
        // first copy the existing ones
        diskState.endpointLinks = oldDiskState.endpointLinks;
    }
    // then append new one
    AdapterUtils.addToEndpointLinks(diskState, context.request.endpointLink);
    diskState.tenantLinks = context.parentCompute.tenantLinks;
    long bLength = 0;
    if (blob instanceof CloudBlob) {
        CloudBlob blobItem = (CloudBlob) blob;
        bLength = blobItem.getProperties().getLength();
    }
    diskState.status = DiskService.DiskStatus.AVAILABLE;
    diskState.capacityMBytes = bLength / B_TO_MB_FACTOR;
    diskState.customProperties = new HashMap<>();
    String storageType = isDisk(diskState.name) ? AZURE_STORAGE_DISKS : AZURE_STORAGE_BLOBS;
    diskState.storageType = storageType;
    if (oldDiskState != null) {
        diskState.tagLinks = (oldDiskState.tagLinks == null) ? new HashSet<>() : oldDiskState.tagLinks;
    } else {
        diskState.tagLinks = new HashSet<>();
    }
    switch(storageType) {
        case AZURE_STORAGE_DISKS:
            if (context.internalVhdTypeTag != null) {
                diskState.tagLinks.add(context.internalVhdTypeTag);
            }
            break;
        case AZURE_STORAGE_BLOBS:
            if (context.internalVhdTypeTag != null) {
                diskState.tagLinks.add(context.internalBlobTypeTag);
            }
            break;
        default:
            break;
    }
    // following two properties are set to defaults - can't retrieve that information from
    // existing calls
    diskState.type = DEFAULT_DISK_TYPE;
    if (oldDiskState != null) {
        if (diskState.storageDescriptionLink == null) {
            diskState.storageDescriptionLink = oldDiskState.storageDescriptionLink;
        }
        diskState.id = oldDiskState.id;
        diskState.documentSelfLink = oldDiskState.documentSelfLink;
        diskState.regionId = oldDiskState.regionId;
    } else {
        diskState.id = canonizeId(blob.getUri().toString());
        StorageAccount storageAccount = context.storageAccountBlobUriMap.get(diskState.id);
        if (storageAccount != null) {
            diskState.regionId = storageAccount.location;
        }
        diskState.documentSelfLink = UUID.randomUUID().toString();
    }
    return diskState;
}