Examples with CloudBlobMetadata com.github.ambry.cloud.CloudBlobMetadata used on opensource projects

Example 26 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobMetadataTest method testDeserUnknownField.

/**
 * Test deserialization of field that was removed from schema
 */
@Test
public void testDeserUnknownField() throws Exception {
    CloudBlobMetadata blobMetadata = new CloudBlobMetadata(blobId, now, -1, 1024, EncryptionOrigin.NONE);
    Map<String, String> propertyMap = blobMetadata.toMap();
    propertyMap.put("cloudBlobName", "1234-" + blobMetadata.getId());
    String serializedString = mapperObj.writeValueAsString(propertyMap);
    CloudBlobMetadata deserBlobMetadata = mapperObj.readValue(serializedString, CloudBlobMetadata.class);
    assertEquals("Expected equality", blobMetadata, deserBlobMetadata);
}

Example 27 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class AzureStorageCompactor method compactPartitionBucketed.

/**
 * Purge the inactive blobs in the specified partition.
 * @param partitionPath the partition to compact.
 * @param fieldName the field name to query on. Allowed values are {@link CloudBlobMetadata#FIELD_DELETION_TIME}
 *                      and {@link CloudBlobMetadata#FIELD_EXPIRATION_TIME}.
 * @param queryStartTime the initial query start time, which will be adjusted as compaction progresses.
 * @param queryEndTime the query end time.
 * @return the number of blobs purged or found.
 * @throws CloudStorageException if the compaction fails.
 */
private int compactPartitionBucketed(String partitionPath, String fieldName, long queryStartTime, long queryEndTime) throws CloudStorageException {
    if (queryEndTime - queryStartTime > TimeUnit.DAYS.toMillis(queryBucketDays)) {
        throw new IllegalArgumentException("Time window is longer than " + queryBucketDays + " days");
    }
    int totalPurged = 0;
    long progressTime = 0;
    while (!isShuttingDown()) {
        // just to use in lambda
        final long newQueryStartTime = queryStartTime;
        List<CloudBlobMetadata> deadBlobs = requestAgent.doWithRetries(() -> getDeadBlobs(partitionPath, fieldName, newQueryStartTime, queryEndTime, queryLimit), "GetDeadBlobs", partitionPath);
        if (deadBlobs.isEmpty()) {
            // If query returned nothing, we can mark progress up to queryEndTime
            progressTime = queryEndTime;
            break;
        }
        if (isShuttingDown()) {
            break;
        }
        totalPurged += requestAgent.doWithRetries(() -> AzureCompactionUtil.purgeBlobs(deadBlobs, azureBlobDataAccessor, azureMetrics, cosmosDataAccessor), "PurgeBlobs", partitionPath);
        vcrMetrics.blobCompactionRate.mark(deadBlobs.size());
        // Adjust startTime for next query
        CloudBlobMetadata lastBlob = deadBlobs.get(deadBlobs.size() - 1);
        progressTime = fieldName.equals(CloudBlobMetadata.FIELD_DELETION_TIME) ? lastBlob.getDeletionTime() : lastBlob.getExpirationTime();
        queryStartTime = progressTime;
        if (deadBlobs.size() < queryLimit) {
            // No more dead blobs to query.
            break;
        }
        if (totalPurged >= purgeLimit) {
            // Reached the purge threshold, give other partitions a chance.
            break;
        }
    }
    if (progressTime > 0) {
        updateCompactionProgress(partitionPath, fieldName, progressTime);
    }
    return totalPurged;
}

Example 28 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobStore method findKey.

@Override
public MessageInfo findKey(StoreKey key) throws StoreException {
    try {
        Map<String, CloudBlobMetadata> cloudBlobMetadataListMap = requestAgent.doWithRetries(() -> cloudDestination.getBlobMetadata(Collections.singletonList((BlobId) key)), "FindKey", partitionId.toPathString());
        CloudBlobMetadata cloudBlobMetadata = cloudBlobMetadataListMap.get(key.getID());
        if (cloudBlobMetadata != null) {
            return new MessageInfo(key, cloudBlobMetadata.getSize(), cloudBlobMetadata.isDeleted(), cloudBlobMetadata.isExpired(), cloudBlobMetadata.isUndeleted(), cloudBlobMetadata.getExpirationTime(), null, (short) cloudBlobMetadata.getAccountId(), (short) cloudBlobMetadata.getContainerId(), cloudBlobMetadata.getLastUpdateTime(), cloudBlobMetadata.getLifeVersion());
        } else {
            throw new StoreException(String.format("FindKey couldn't find key: %s", key), StoreErrorCodes.ID_Not_Found);
        }
    } catch (CloudStorageException e) {
        throw new StoreException(e, StoreErrorCodes.IOError);
    }
}

Example 29 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobStore method get.

@Override
public StoreInfo get(List<? extends StoreKey> ids, EnumSet<StoreGetOptions> storeGetOptions) throws StoreException {
    checkStarted();
    checkStoreKeyDuplicates(ids);
    List<CloudMessageReadSet.BlobReadInfo> blobReadInfos = new ArrayList<>(ids.size());
    List<MessageInfo> messageInfos = new ArrayList<>(ids.size());
    try {
        List<BlobId> blobIdList = ids.stream().map(key -> (BlobId) key).collect(Collectors.toList());
        Map<String, CloudBlobMetadata> cloudBlobMetadataListMap = requestAgent.doWithRetries(() -> cloudDestination.getBlobMetadata(blobIdList), "GetBlobMetadata", partitionId.toPathString());
        // Throw StoreException with ID_Not_Found if cloudBlobMetadataListMap size is less than expected.
        if (cloudBlobMetadataListMap.size() < blobIdList.size()) {
            Set<BlobId> missingBlobs = blobIdList.stream().filter(blobId -> !cloudBlobMetadataListMap.containsKey(blobId)).collect(Collectors.toSet());
            throw new StoreException("Some of the keys were missing in the cloud metadata store: " + missingBlobs, StoreErrorCodes.ID_Not_Found);
        }
        long currentTimeStamp = System.currentTimeMillis();
        // Validate cloud meta data, may throw StoreException with ID_Deleted, TTL_Expired and Authorization_Failure
        validateCloudMetadata(cloudBlobMetadataListMap, storeGetOptions, currentTimeStamp, ids);
        for (BlobId blobId : blobIdList) {
            CloudBlobMetadata blobMetadata = cloudBlobMetadataListMap.get(blobId.getID());
            // TODO: need to add ttlUpdated to CloudBlobMetadata so we can use it here
            // For now, set ttlUpdated = true for all permanent blobs, so the correct ttl
            // is applied by GetOperation.
            boolean ttlUpdated = blobMetadata.getExpirationTime() == Utils.Infinite_Time;
            boolean deleted = blobMetadata.getDeletionTime() != Utils.Infinite_Time;
            MessageInfo messageInfo = new MessageInfo(blobId, blobMetadata.getSize(), deleted, ttlUpdated, blobMetadata.isUndeleted(), blobMetadata.getExpirationTime(), null, (short) blobMetadata.getAccountId(), (short) blobMetadata.getContainerId(), getOperationTime(blobMetadata), blobMetadata.getLifeVersion());
            messageInfos.add(messageInfo);
            blobReadInfos.add(new CloudMessageReadSet.BlobReadInfo(blobMetadata, blobId));
        }
    } catch (CloudStorageException e) {
        if (e.getCause() instanceof StoreException) {
            throw (StoreException) e.getCause();
        } else {
            throw new StoreException(e, StoreErrorCodes.IOError);
        }
    }
    CloudMessageReadSet messageReadSet = new CloudMessageReadSet(blobReadInfos, this);
    return new StoreInfo(messageReadSet, messageInfos);
}

Example 30 with CloudBlobMetadata

use of com.github.ambry.cloud.CloudBlobMetadata in project ambry by linkedin.

the class CloudBlobStore method findEntriesSince.

@Override
public FindInfo findEntriesSince(FindToken token, long maxTotalSizeOfEntries, String hostname, String remoteReplicaPath) throws StoreException {
    try {
        FindResult findResult = requestAgent.doWithRetries(() -> cloudDestination.findEntriesSince(partitionId.toPathString(), token, maxTotalSizeOfEntries), "FindEntriesSince", partitionId.toPathString());
        if (findResult.getMetadataList().isEmpty()) {
            return new FindInfo(Collections.emptyList(), findResult.getUpdatedFindToken());
        }
        List<MessageInfo> messageEntries = new ArrayList<>();
        for (CloudBlobMetadata metadata : findResult.getMetadataList()) {
            messageEntries.add(getMessageInfoFromMetadata(metadata));
        }
        return new FindInfo(messageEntries, findResult.getUpdatedFindToken());
    } catch (CloudStorageException | IOException ex) {
        throw new StoreException(ex, StoreErrorCodes.IOError);
    }
}