Examples with AzureException org.apache.samza.AzureException used on opensource projects

Example 6 with AzureException

use of org.apache.samza.AzureException in project samza by apache.

the class AzureBlobOutputStream method close.

/**
 * This api waits for all pending upload (stageBlock task) futures to finish.
 * It then synchronously commits the list of blocks to persist the actual blob on storage.
 * Note: this method does not invoke flush and flush has to be explicitly called before close.
 * @throws IllegalStateException when
 *       - when closing an already closed stream
 * @throws RuntimeException when
 *       - byteArrayOutputStream.close fails or
 *       - any of the pending uploads fails or
 *       - blob's commitBlockList fails
 * throws ClassNotFoundException or IllegalAccessException or InstantiationException
 *       - while creating an instance of BlobMetadataGenerator
 */
@Override
public synchronized void close() {
    if (isClosed) {
        LOG.info("{}: already closed", blobAsyncClient.getBlobUrl().toString());
        return;
    }
    LOG.info("{}: Close", blobAsyncClient.getBlobUrl().toString());
    try {
        if (byteArrayOutputStream.isPresent()) {
            byteArrayOutputStream.get().close();
        }
        if (blockList.size() == 0) {
            return;
        }
        CompletableFuture<Void> future = CompletableFuture.allOf(pendingUpload.toArray(new CompletableFuture[0]));
        LOG.info("Closing blob: {} PendingUpload:{} ", blobAsyncClient.getBlobUrl().toString(), pendingUpload.size());
        future.get((long) flushTimeoutMs, TimeUnit.MILLISECONDS);
        LOG.info("For blob: {} committing blockList size:{}", blobAsyncClient.getBlobUrl().toString(), blockList.size());
        metrics.updateAzureCommitMetrics();
        BlobMetadataGenerator blobMetadataGenerator = getBlobMetadataGenerator();
        commitBlob(blockList, blobMetadataGenerator.getBlobMetadata(new BlobMetadataContext(streamName, totalUploadedBlockSize, totalNumberOfRecordsInBlob)));
    } catch (Exception e) {
        String msg = String.format("Close blob %s failed with exception. Total pending sends %d", blobAsyncClient.getBlobUrl().toString(), pendingUpload.size());
        throw new AzureException(msg, e);
    } finally {
        clearAndMarkClosed();
    }
}

Example 7 with AzureException

use of org.apache.samza.AzureException in project samza by apache.

the class AzureLock method lock.

/**
 * Tries to acquire a lock in order to create intermediate streams. On failure to acquire lock, it keeps trying until the lock times out.
 * The lock is acquired when the blob is leased successfully.
 * @param timeout Duration after which timeout occurs.
 * @return true if the lock was acquired successfully, false if lock acquire operation is unsuccessful even after subsequent tries within the timeout range.
 */
@Override
public boolean lock(Duration timeout) {
    // Start timer for timeout
    long startTime = System.currentTimeMillis();
    long lockTimeout = timeout.toMillis();
    Random random = new Random();
    while ((System.currentTimeMillis() - startTime) < lockTimeout) {
        try {
            leaseId.getAndSet(leaseBlobManager.acquireLease(LEASE_TIME_IN_SEC, leaseId.get()));
        } catch (AzureException e) {
            return false;
        }
        if (leaseId.get() != null) {
            LOG.info("Acquired lock!");
            hasLock.getAndSet(true);
            return true;
        } else {
            try {
                Thread.sleep(random.nextInt(1000));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
            LOG.info("Trying to acquire lock again...");
        }
    }
    return false;
}

Example 8 with AzureException

use of org.apache.samza.AzureException in project samza by apache.

the class AzureCheckpointManager method writeCheckpoint.

@Override
public void writeCheckpoint(TaskName taskName, Checkpoint checkpoint) {
    Preconditions.checkArgument(checkpoint instanceof CheckpointV1, "Only CheckpointV1 could be written to Azure");
    if (!taskNames.contains(taskName)) {
        throw new SamzaException("writing checkpoint of unregistered task");
    }
    TableBatchOperation batchOperation = new TableBatchOperation();
    Iterator<Map.Entry<SystemStreamPartition, String>> iterator = checkpoint.getOffsets().entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<SystemStreamPartition, String> entry = iterator.next();
        SystemStreamPartition ssp = entry.getKey();
        String offset = entry.getValue();
        String partitionKey = taskName.toString();
        checkValidKey(partitionKey, "Taskname");
        String rowKey = serializeSystemStreamPartition(ssp);
        checkValidKey(rowKey, "SystemStreamPartition");
        // Create table entity
        TaskCheckpointEntity taskCheckpoint = new TaskCheckpointEntity(partitionKey, rowKey, offset);
        // Add to batch operation
        batchOperation.insertOrReplace(taskCheckpoint);
        // Execute when batch reaches capacity or this is the last item
        if (batchOperation.size() >= MAX_WRITE_BATCH_SIZE || !iterator.hasNext()) {
            try {
                cloudTable.execute(batchOperation);
            } catch (StorageException e) {
                LOG.error("Executing batch failed for task: {}", taskName);
                throw new AzureException(e);
            }
            batchOperation.clear();
        }
    }
}

Example 9 with AzureException

use of org.apache.samza.AzureException in project samza by apache.

the class AzureCheckpointManager method deleteEntities.

private void deleteEntities(Iterator<TaskCheckpointEntity> entitiesToDelete) {
    TableBatchOperation batchOperation = new TableBatchOperation();
    while (entitiesToDelete.hasNext()) {
        TaskCheckpointEntity entity = entitiesToDelete.next();
        // Add to batch operation
        batchOperation.delete(entity);
        // Execute when batch reaches capacity or when this is the last item
        if (batchOperation.size() >= MAX_WRITE_BATCH_SIZE || !entitiesToDelete.hasNext()) {
            try {
                cloudTable.execute(batchOperation);
            } catch (StorageException e) {
                LOG.error("Executing batch failed for deleting checkpoints");
                throw new AzureException(e);
            }
            batchOperation.clear();
        }
    }
}

Example 10 with AzureException

use of org.apache.samza.AzureException in project samza by apache.

the class AzureCheckpointManager method start.

@Override
public void start() {
    try {
        // Create the table if it doesn't exist.
        cloudTable = azureClient.getTableClient().getTableReference(jobTableName);
        cloudTable.createIfNotExists();
    } catch (URISyntaxException e) {
        LOG.error("Connection string {} specifies an invalid URI while creating checkpoint table.", storageConnectionString);
        throw new AzureException(e);
    } catch (StorageException e) {
        LOG.error("Azure Storage failed when creating checkpoint table", e);
        throw new AzureException(e);
    }
}