Examples with BlobMetadataGenerator org.apache.samza.system.azureblob.utils.BlobMetadataGenerator used on opensource projects

Example 1 with BlobMetadataGenerator

use of org.apache.samza.system.azureblob.utils.BlobMetadataGenerator in project samza by apache.

the class AzureBlobOutputStream method close.

/**
 * This api waits for all pending upload (stageBlock task) futures to finish.
 * It then synchronously commits the list of blocks to persist the actual blob on storage.
 * Note: this method does not invoke flush and flush has to be explicitly called before close.
 * @throws IllegalStateException when
 *       - when closing an already closed stream
 * @throws RuntimeException when
 *       - byteArrayOutputStream.close fails or
 *       - any of the pending uploads fails or
 *       - blob's commitBlockList fails
 * throws ClassNotFoundException or IllegalAccessException or InstantiationException
 *       - while creating an instance of BlobMetadataGenerator
 */
@Override
public synchronized void close() {
    if (isClosed) {
        LOG.info("{}: already closed", blobAsyncClient.getBlobUrl().toString());
        return;
    }
    LOG.info("{}: Close", blobAsyncClient.getBlobUrl().toString());
    try {
        if (byteArrayOutputStream.isPresent()) {
            byteArrayOutputStream.get().close();
        }
        if (blockList.size() == 0) {
            return;
        }
        CompletableFuture<Void> future = CompletableFuture.allOf(pendingUpload.toArray(new CompletableFuture[0]));
        LOG.info("Closing blob: {} PendingUpload:{} ", blobAsyncClient.getBlobUrl().toString(), pendingUpload.size());
        future.get((long) flushTimeoutMs, TimeUnit.MILLISECONDS);
        LOG.info("For blob: {} committing blockList size:{}", blobAsyncClient.getBlobUrl().toString(), blockList.size());
        metrics.updateAzureCommitMetrics();
        BlobMetadataGenerator blobMetadataGenerator = getBlobMetadataGenerator();
        commitBlob(blockList, blobMetadataGenerator.getBlobMetadata(new BlobMetadataContext(streamName, totalUploadedBlockSize, totalNumberOfRecordsInBlob)));
    } catch (Exception e) {
        String msg = String.format("Close blob %s failed with exception. Total pending sends %d", blobAsyncClient.getBlobUrl().toString(), pendingUpload.size());
        throw new AzureException(msg, e);
    } finally {
        clearAndMarkClosed();
    }
}