Search in sources :

Example 1 with TagSet

use of com.amazonaws.services.s3.model.TagSet in project photon-model by vmware.

the class TestAWSEnumerationTask method createS3BucketAndTags.

private void createS3BucketAndTags(Map<String, String> tags) {
    this.s3Client.createBucket(TEST_BUCKET_NAME);
    TagSet tagSet = new TagSet(tags);
    BucketTaggingConfiguration bucketTaggingConfiguration = new BucketTaggingConfiguration(Collections.singletonList(tagSet));
    this.s3Client.setBucketTaggingConfiguration(TEST_BUCKET_NAME, bucketTaggingConfiguration);
}
Also used : BucketTaggingConfiguration(com.amazonaws.services.s3.model.BucketTaggingConfiguration) TagSet(com.amazonaws.services.s3.model.TagSet)

Example 2 with TagSet

use of com.amazonaws.services.s3.model.TagSet in project photon-model by vmware.

the class AWSS3StorageEnumerationAdapterService method enumerateTags.

/**
 * Calls getBucketTaggingConfiguration() on every bucket to enumerate bucket tags.
 * getBucketTaggingConfiguration() method is region aware and only returns valid results when we
 * call it with the client region same as the S3 bucket region. Since S3 bucket region is not
 * available through API, when a new bucket is discovered, we try to get tags for it by calling
 * getBucketTaggingConfiguration() with client in every AWS region. We then store the client region
 * for which we received successful response as region in DiskState for that S3 bucket. This region
 * in DiskState is then used for any subsequent calls for enumerating tags.
 */
private void enumerateTags(S3StorageEnumerationContext aws, S3StorageEnumerationSubStage next) {
    OperationContext operationContext = OperationContext.getOperationContext();
    this.executorService.submit(() -> {
        OperationContext.restoreOperationContext(operationContext);
        List<DeferredResult<S3ClientHandler>> s3ClientsEnumeratePreviousBuckets = new ArrayList<>();
        // Get a client for that region and make a call to enumerate tags.
        for (Map.Entry<String, DiskState> entry : aws.diskStatesToBeUpdatedByBucketName.entrySet()) {
            // with valid region in subsequent enumeration runs.
            if (entry.getValue().regionId != null) {
                aws.regionsByBucketName.put(entry.getValue().id, entry.getValue().regionId);
            } else {
                logWarning("Null region found in S3 diskState");
                Operation.createDelete(aws.service.getHost(), entry.getValue().documentSelfLink).setReferer(aws.service.getUri()).setBody(getDeletionState(Utils.getNowMicrosUtc())).setCompletion((o, e) -> {
                    if (e != null) {
                        logWarning("Exception deleting diskState with null " + "region [ex=%s]", e.getMessage());
                        return;
                    }
                    logWarning("Deleted diskState with null region [diskState=%s]", Utils.toJsonHtml(entry.getValue()));
                }).sendWith(aws.service);
                continue;
            }
            s3ClientsEnumeratePreviousBuckets.add(aws.clientManager.getOrCreateS3ClientAsync(aws.endpointAuth, entry.getValue().regionId, aws.service).thenApply(s3Client -> {
                S3ClientHandler s3ClientHandler = new S3ClientHandler();
                s3ClientHandler.s3Client = s3Client;
                s3ClientHandler.bucketName = entry.getKey();
                s3ClientHandler.diskState = entry.getValue();
                return s3ClientHandler;
            }));
        }
        // Handler to enumerate new buckets not previously enumerated.
        BiConsumer<List<S3ClientHandler>, Throwable> enumerateNewBuckets = (s3ClientHandlers, throwable) -> {
            OperationContext.restoreOperationContext(operationContext);
            for (Bucket bucket : aws.bucketsToBeCreated) {
                for (S3ClientHandler s3ClientHandler : s3ClientHandlers) {
                    try {
                        BucketTaggingConfiguration bucketTaggingConfiguration = s3ClientHandler.s3Client.getBucketTaggingConfiguration(bucket.getName());
                        aws.regionsByBucketName.put(bucket.getName(), s3ClientHandler.region.getName());
                        if (bucketTaggingConfiguration != null) {
                            aws.tagsByBucketName.put(bucket.getName(), new ConcurrentHashMap<>());
                            bucketTaggingConfiguration.getAllTagSets().forEach(tagSet -> aws.tagsByBucketName.get(bucket.getName()).putAll(tagSet.getAllTags()));
                        }
                        break;
                    } catch (Exception e) {
                        // getbucketTaggingConfiguration().
                        if (e instanceof AmazonS3Exception && (((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_MOVED_PERM || ((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_FORBIDDEN || ((AmazonS3Exception) e).getStatusCode() == Operation.STATUS_CODE_BAD_REQUEST)) {
                            continue;
                        } else {
                            logSevere("Exception enumerating tags for S3 bucket with unknown region " + "[endpoint=%s] [region=%s] [ex=%s]", aws.request.original.endpointLink, s3ClientHandler.region.getName(), e.getMessage());
                            continue;
                        }
                    }
                }
            }
            // Once completed, move to next stage.
            aws.subStage = next;
            handleReceivedEnumerationData(aws);
        };
        // Handler to enumerate tags for buckets already previously enumerated.
        BiConsumer<List<S3ClientHandler>, Throwable> enumeratePreviousBuckets = (s3ClientHandlers, ignored) -> {
            OperationContext.restoreOperationContext(operationContext);
            for (S3ClientHandler s3ClientHandler : s3ClientHandlers) {
                try {
                    BucketTaggingConfiguration bucketTaggingConfiguration = s3ClientHandler.s3Client.getBucketTaggingConfiguration(s3ClientHandler.bucketName);
                    if (bucketTaggingConfiguration != null) {
                        aws.tagsByBucketName.put(s3ClientHandler.bucketName, new ConcurrentHashMap<>());
                        bucketTaggingConfiguration.getAllTagSets().forEach(tagSet -> aws.tagsByBucketName.get(s3ClientHandler.bucketName).putAll(tagSet.getAllTags()));
                    }
                } catch (Exception e) {
                    logSevere("Exception enumerating tags for S3 bucket with known region " + "[endpoint=%s] [bucketName=%s - %s] [region=%s] [ex=%s]", aws.request.original.endpointLink, s3ClientHandler.bucketName, s3ClientHandler.diskState.id, s3ClientHandler.diskState.regionId, e.getMessage());
                }
            }
            // For remaining buckets, they have yet to be enumerated. Brute force and try to
            // enumerate tags for these buckets over every region until we find the correct
            // region and then store it in DiskState for future reference.
            List<DeferredResult<S3ClientHandler>> s3ClientBucketsToEnumerate = new ArrayList<>();
            for (Regions region : Regions.values()) {
                // Get an s3 client in the region asynchronously. Once all completed, these
                // clients will be used to test each bucket.
                s3ClientBucketsToEnumerate.add(aws.clientManager.getOrCreateS3ClientAsync(aws.endpointAuth, region.getName(), aws.service).thenApply(s3Client -> {
                    S3ClientHandler s3ClientHandler = new S3ClientHandler();
                    s3ClientHandler.s3Client = s3Client;
                    s3ClientHandler.region = region;
                    return s3ClientHandler;
                }));
            }
            // Once finished, attempt to enumerate each of the "new" buckets.
            DeferredResult.allOf(s3ClientBucketsToEnumerate).whenComplete(enumerateNewBuckets);
        };
        // Enumerate tags of previously enumerated buckets first
        DeferredResult.allOf(s3ClientsEnumeratePreviousBuckets).whenComplete(enumeratePreviousBuckets);
    });
}
Also used : OperationContext(com.vmware.xenon.common.OperationContext) BUCKET_OWNER_NAME(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.BUCKET_OWNER_NAME) BucketTaggingConfiguration(com.amazonaws.services.s3.model.BucketTaggingConfiguration) AWSResourceType(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWSResourceType) QueryTask(com.vmware.xenon.services.common.QueryTask) Utils(com.vmware.xenon.common.Utils) STORAGE_TYPE_S3(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_S3) Map(java.util.Map) AWSClientManager(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager) StatelessService(com.vmware.xenon.common.StatelessService) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) AdapterUtils.getDeletionState(com.vmware.photon.controller.model.adapters.util.AdapterUtils.getDeletionState) List(java.util.List) TagService(com.vmware.photon.controller.model.resources.TagService) DeferredResult(com.vmware.xenon.common.DeferredResult) TAG_KEY_TYPE(com.vmware.photon.controller.model.constants.PhotonModelConstants.TAG_KEY_TYPE) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) DiskService(com.vmware.photon.controller.model.resources.DiskService) OperationContext(com.vmware.xenon.common.OperationContext) HashMap(java.util.HashMap) PhotonModelUtils(com.vmware.photon.controller.model.resources.util.PhotonModelUtils) TagsUtil(com.vmware.photon.controller.model.adapters.util.TagsUtil) ArrayList(java.util.ArrayList) Regions(com.amazonaws.regions.Regions) HashSet(java.util.HashSet) AWSConstants(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants) TagState(com.vmware.photon.controller.model.resources.TagService.TagState) ServiceStateCollectionUpdateRequest(com.vmware.xenon.common.ServiceStateCollectionUpdateRequest) AuthCredentialsService(com.vmware.xenon.services.common.AuthCredentialsService) Query(com.vmware.xenon.services.common.QueryTask.Query) AdapterUtils.createPostOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPostOperation) BiConsumer(java.util.function.BiConsumer) EnumerationAction(com.vmware.photon.controller.model.adapterapi.EnumerationAction) AwsClientType(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AwsClientType) ExecutorService(java.util.concurrent.ExecutorService) AWSConstants.getQueryResultLimit(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.getQueryResultLimit) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) Bucket(com.amazonaws.services.s3.model.Bucket) TimeUnit(java.util.concurrent.TimeUnit) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) ComputeEnumerateAdapterRequest(com.vmware.photon.controller.model.adapters.util.ComputeEnumerateAdapterRequest) AWSClientManagerFactory(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory) Collections(java.util.Collections) AWSUriPaths(com.vmware.photon.controller.model.adapters.awsadapter.AWSUriPaths) OperationJoin(com.vmware.xenon.common.OperationJoin) PhotonModelUriUtils.createInventoryUri(com.vmware.photon.controller.model.util.PhotonModelUriUtils.createInventoryUri) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) ArrayList(java.util.ArrayList) Regions(com.amazonaws.regions.Regions) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) BucketTaggingConfiguration(com.amazonaws.services.s3.model.BucketTaggingConfiguration) Bucket(com.amazonaws.services.s3.model.Bucket) List(java.util.List) ArrayList(java.util.ArrayList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) DeferredResult(com.vmware.xenon.common.DeferredResult)

Aggregations

BucketTaggingConfiguration (com.amazonaws.services.s3.model.BucketTaggingConfiguration)2 Regions (com.amazonaws.regions.Regions)1 AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)1 AmazonS3Exception (com.amazonaws.services.s3.model.AmazonS3Exception)1 Bucket (com.amazonaws.services.s3.model.Bucket)1 TagSet (com.amazonaws.services.s3.model.TagSet)1 EnumerationAction (com.vmware.photon.controller.model.adapterapi.EnumerationAction)1 AWSConstants (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants)1 AWSResourceType (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWSResourceType)1 AwsClientType (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AwsClientType)1 BUCKET_OWNER_NAME (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.BUCKET_OWNER_NAME)1 STORAGE_TYPE_S3 (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.STORAGE_TYPE_S3)1 AWSConstants.getQueryResultLimit (com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.getQueryResultLimit)1 AWSUriPaths (com.vmware.photon.controller.model.adapters.awsadapter.AWSUriPaths)1 AWSClientManager (com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager)1 AWSClientManagerFactory (com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory)1 AdapterUtils (com.vmware.photon.controller.model.adapters.util.AdapterUtils)1 AdapterUtils.createPostOperation (com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPostOperation)1 AdapterUtils.getDeletionState (com.vmware.photon.controller.model.adapters.util.AdapterUtils.getDeletionState)1 ComputeEnumerateAdapterRequest (com.vmware.photon.controller.model.adapters.util.ComputeEnumerateAdapterRequest)1