Search in sources :

Example 46 with Tag

use of com.amazonaws.services.s3.model.Tag in project herd by FINRAOS.

the class S3DaoImpl method tagObjects.

@Override
public void tagObjects(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final Tag tag) {
    LOGGER.info("Tagging objects in S3... s3BucketName=\"{}\" s3KeyCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\"", s3FileTransferRequestParamsDto.getS3BucketName(), s3FileTransferRequestParamsDto.getFiles().size(), tag.getKey(), tag.getValue());
    if (!CollectionUtils.isEmpty(s3FileTransferRequestParamsDto.getFiles())) {
        // Initialize a key value pair for the error message in the catch block.
        String s3Key = s3FileTransferRequestParamsDto.getFiles().get(0).getPath().replaceAll("\\\\", "/");
        // Amazon S3 client to access S3 objects.
        AmazonS3Client s3Client = null;
        // Amazon S3 client for S3 object tagging.
        AmazonS3Client s3ObjectTaggerClient = null;
        try {
            // Create an S3 client to access S3 objects.
            s3Client = getAmazonS3(s3FileTransferRequestParamsDto);
            // Create an S3 client for S3 object tagging.
            s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto);
            // Create a get object tagging request.
            GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null);
            // Create a restore object request.
            SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest(s3FileTransferRequestParamsDto.getS3BucketName(), null, null);
            for (File file : s3FileTransferRequestParamsDto.getFiles()) {
                // Prepare an S3 key.
                s3Key = file.getPath().replaceAll("\\\\", "/");
                // Retrieve the current tagging information for the S3 key.
                getObjectTaggingRequest.setKey(s3Key);
                GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(getObjectTaggingRequest, s3Client);
                // Update the list of tags to include the specified S3 object tag.
                List<Tag> updatedTags = new ArrayList<>();
                updatedTags.add(tag);
                if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet())) {
                    for (Tag currentTag : getObjectTaggingResult.getTagSet()) {
                        if (!StringUtils.equals(tag.getKey(), currentTag.getKey())) {
                            updatedTags.add(currentTag);
                        }
                    }
                }
                // Update the tagging information.
                setObjectTaggingRequest.setKey(s3Key);
                setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags));
                s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient);
            }
        } catch (Exception e) {
            throw new IllegalStateException(String.format("Failed to tag S3 object with \"%s\" key in \"%s\" bucket. Reason: %s", s3Key, s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e);
        } finally {
            if (s3Client != null) {
                s3Client.shutdown();
            }
            if (s3ObjectTaggerClient != null) {
                s3ObjectTaggerClient.shutdown();
            }
        }
    }
}
Also used : GetObjectTaggingRequest(com.amazonaws.services.s3.model.GetObjectTaggingRequest) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) SetObjectTaggingRequest(com.amazonaws.services.s3.model.SetObjectTaggingRequest) ArrayList(java.util.ArrayList) Tag(com.amazonaws.services.s3.model.Tag) File(java.io.File) ObjectTagging(com.amazonaws.services.s3.model.ObjectTagging) MultiObjectDeleteException(com.amazonaws.services.s3.model.MultiObjectDeleteException) ObjectNotFoundException(org.finra.herd.model.ObjectNotFoundException) AmazonServiceException(com.amazonaws.AmazonServiceException) AmazonClientException(com.amazonaws.AmazonClientException) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) GetObjectTaggingResult(com.amazonaws.services.s3.model.GetObjectTaggingResult)

Example 47 with Tag

use of com.amazonaws.services.s3.model.Tag in project herd by FINRAOS.

the class S3DaoTest method testTagObjectsOtherTagKeyAlreadyExists.

@Test
public void testTagObjectsOtherTagKeyAlreadyExists() {
    // Create two S3 object tags having different tag keys.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2));
    // Put a file in S3 that is already tagged with the first S3 object tag.
    PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Arrays.asList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);
    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Arrays.asList(tags.get(0)), getObjectTaggingResult.getTagSet());
    // Tag the S3 file with the second S3 object tag.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);
    params.setFiles(Arrays.asList(new File(TARGET_S3_KEY)));
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), tags.get(1));
    // Validate that the S3 object is now tagged with both tags.
    getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size());
    assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags));
}
Also used : GetObjectTaggingRequest(com.amazonaws.services.s3.model.GetObjectTaggingRequest) S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) ByteArrayInputStream(java.io.ByteArrayInputStream) Tag(com.amazonaws.services.s3.model.Tag) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) File(java.io.File) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) ObjectTagging(com.amazonaws.services.s3.model.ObjectTagging) GetObjectTaggingResult(com.amazonaws.services.s3.model.GetObjectTaggingResult) Test(org.junit.Test)

Example 48 with Tag

use of com.amazonaws.services.s3.model.Tag in project herd by FINRAOS.

the class S3DaoTest method testTagObjectsAmazonServiceException.

@Test
public void testTagObjectsAmazonServiceException() {
    // Try to retrieve S3 object metadata when AmazonServiceException is expected tpo be thrown..
    try {
        S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
        params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);
        params.setFiles(Arrays.asList(new File(TARGET_S3_KEY)));
        s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
        fail();
    } catch (IllegalStateException e) {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key in \"%s\" bucket. " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", TARGET_S3_KEY, MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR), e.getMessage());
    }
}
Also used : S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) Tag(com.amazonaws.services.s3.model.Tag) File(java.io.File) Test(org.junit.Test)

Example 49 with Tag

use of com.amazonaws.services.s3.model.Tag in project photon-model by vmware.

the class AWSComputeStateCreationAdapterService method updateTagLinks.

/**
 * Updates tag links of existing computes using TagsUtil.
 */
private DeferredResult<AWSComputeStateCreationContext> updateTagLinks(AWSComputeStateCreationContext context) {
    if (context.request.instancesToBeUpdated == null || context.request.instancesToBeUpdated.size() == 0) {
        logFine(() -> "No local compute states to be updated so there are no tags to update.");
        return DeferredResult.completed(context);
    } else {
        List<DeferredResult<Set<String>>> updateCSTagLinksOps = new ArrayList<>();
        for (String instanceId : context.request.instancesToBeUpdated.keySet()) {
            Instance instance = context.request.instancesToBeUpdated.get(instanceId);
            ComputeState existingComputeState = context.request.computeStatesToBeUpdated.get(instanceId);
            Map<String, String> remoteTags = new HashMap<>();
            for (Tag awsInstanceTag : instance.getTags()) {
                if (!awsInstanceTag.getKey().equals(AWSConstants.AWS_TAG_NAME)) {
                    remoteTags.put(awsInstanceTag.getKey(), awsInstanceTag.getValue());
                }
            }
            updateCSTagLinksOps.add(updateLocalTagStates(this, existingComputeState, remoteTags, null));
        }
        return DeferredResult.allOf(updateCSTagLinksOps).thenApply(gnore -> context);
    }
}
Also used : AWSEnumerationUtils.mapInstanceToComputeState(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.mapInstanceToComputeState) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) Instance(com.amazonaws.services.ec2.model.Instance) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Tag(com.amazonaws.services.ec2.model.Tag) DeferredResult(com.vmware.xenon.common.DeferredResult)

Example 50 with Tag

use of com.amazonaws.services.s3.model.Tag in project photon-model by vmware.

the class AWSComputeStateCreationAdapterService method createInternalTypeTags.

private void createInternalTypeTags(AWSComputeStateCreationContext context, AWSComputeStateCreationStage next) {
    // Go over the list of internal tags to be created. Find whatever already does not have an
    // associated tag state and create an operation for its creation.
    List<Operation> joinOperations = new ArrayList<>();
    for (String resourceType : internalTagList) {
        TagState typeTag = newTagState(TAG_KEY_TYPE, resourceType, false, context.request.tenantLinks);
        Operation op = Operation.createPost(this, TagService.FACTORY_LINK).setBody(typeTag);
        joinOperations.add(op);
    }
    OperationJoin.create(joinOperations).setCompletion((ops, exs) -> {
        if (exs != null) {
            exs.values().forEach(ex -> logWarning(() -> String.format("Error creating internal tag%s", ex.getMessage())));
            context.creationStage = next;
            handleComputeStateCreateOrUpdate(context);
            return;
        }
        for (String internalTagValue : internalTagList) {
            TagState tagState = newTagState(TAG_KEY_TYPE, internalTagValue, false, context.request.tenantLinks);
            context.internalTagLinksMap.put(tagState.value, new HashSet<>(Arrays.asList(tagState.documentSelfLink)));
        }
        context.creationStage = next;
        handleComputeStateCreateOrUpdate(context);
    }).sendWith(this);
}
Also used : AdapterUtils.createDeleteOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createDeleteOperation) AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) Arrays(java.util.Arrays) QueryTask(com.vmware.xenon.services.common.QueryTask) InstanceDescKey(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.InstanceDescKey) AWSEnumerationUtils(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils) StringUtils(org.apache.commons.lang3.StringUtils) AWSEnumerationUtils.getKeyForComputeDescriptionFromCD(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.getKeyForComputeDescriptionFromCD) Utils(com.vmware.xenon.common.Utils) Map(java.util.Map) GroupIdentifier(com.amazonaws.services.ec2.model.GroupIdentifier) AdapterUtils.createPatchOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPatchOperation) URI(java.net.URI) TagsUtil.newTagState(com.vmware.photon.controller.model.adapters.util.TagsUtil.newTagState) AWSClientManager(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManager) ComputeDescription(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription) NetworkInterfaceState(com.vmware.photon.controller.model.resources.NetworkInterfaceService.NetworkInterfaceState) StatelessService(com.vmware.xenon.common.StatelessService) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) DiskState(com.vmware.photon.controller.model.resources.DiskService.DiskState) UUID(java.util.UUID) AdapterUtils.getDeletionState(com.vmware.photon.controller.model.adapters.util.AdapterUtils.getDeletionState) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) NetworkInterfaceService(com.vmware.photon.controller.model.resources.NetworkInterfaceService) TagService(com.vmware.photon.controller.model.resources.TagService) Tag(com.amazonaws.services.ec2.model.Tag) DeferredResult(com.vmware.xenon.common.DeferredResult) UriUtils(com.vmware.xenon.common.UriUtils) TAG_KEY_TYPE(com.vmware.photon.controller.model.constants.PhotonModelConstants.TAG_KEY_TYPE) ComputeService(com.vmware.photon.controller.model.resources.ComputeService) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) AWSEnumerationUtils.mapInstanceToComputeState(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.mapInstanceToComputeState) ZoneData(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.ZoneData) InstanceNetworkInterface(com.amazonaws.services.ec2.model.InstanceNetworkInterface) AWSResourceType.ec2_instance(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWSResourceType.ec2_instance) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) AWSConstants(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants) TagState(com.vmware.photon.controller.model.resources.TagService.TagState) AWSResourceType.ec2_net_interface(com.vmware.photon.controller.model.adapters.awsadapter.AWSConstants.AWSResourceType.ec2_net_interface) ServiceStateCollectionUpdateRequest(com.vmware.xenon.common.ServiceStateCollectionUpdateRequest) Query(com.vmware.xenon.services.common.QueryTask.Query) TagsUtil.updateLocalTagStates(com.vmware.photon.controller.model.adapters.util.TagsUtil.updateLocalTagStates) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) AdapterUtils.createPostOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPostOperation) AWSEnumerationUtils.getRepresentativeListOfCDsFromInstanceList(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.getRepresentativeListOfCDsFromInstanceList) BiConsumer(java.util.function.BiConsumer) Instance(com.amazonaws.services.ec2.model.Instance) AWSSecurityGroupEnumerationResponse(com.vmware.photon.controller.model.adapters.awsadapter.enumeration.AWSSecurityGroupEnumerationAdapterService.AWSSecurityGroupEnumerationResponse) ResourceState(com.vmware.photon.controller.model.resources.ResourceState) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) AWSNetworkEnumerationResponse(com.vmware.photon.controller.model.adapters.awsadapter.enumeration.AWSNetworkStateEnumerationAdapterService.AWSNetworkEnumerationResponse) TimeUnit(java.util.concurrent.TimeUnit) AWSEnumerationUtils.getCDsRepresentingVMsInLocalSystemCreatedByEnumerationQuery(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSEnumerationUtils.getCDsRepresentingVMsInLocalSystemCreatedByEnumerationQuery) AWSClientManagerFactory(com.vmware.photon.controller.model.adapters.awsadapter.util.AWSClientManagerFactory) Collections(java.util.Collections) AWSUriPaths(com.vmware.photon.controller.model.adapters.awsadapter.AWSUriPaths) OperationJoin(com.vmware.xenon.common.OperationJoin) ArrayList(java.util.ArrayList) AdapterUtils.createDeleteOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createDeleteOperation) AdapterUtils.createPatchOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPatchOperation) AdapterUtils.createPostOperation(com.vmware.photon.controller.model.adapters.util.AdapterUtils.createPostOperation) Operation(com.vmware.xenon.common.Operation) TagsUtil.newTagState(com.vmware.photon.controller.model.adapters.util.TagsUtil.newTagState) TagState(com.vmware.photon.controller.model.resources.TagService.TagState) HashSet(java.util.HashSet)

Aggregations

Tag (com.amazonaws.services.ec2.model.Tag)38 ArrayList (java.util.ArrayList)30 Tag (com.amazonaws.services.s3.model.Tag)19 HashMap (java.util.HashMap)18 Test (org.junit.Test)17 List (java.util.List)16 Instance (com.amazonaws.services.ec2.model.Instance)15 S3FileTransferRequestParamsDto (org.finra.herd.model.dto.S3FileTransferRequestParamsDto)14 Map (java.util.Map)12 HashSet (java.util.HashSet)10 GetObjectTaggingRequest (com.amazonaws.services.s3.model.GetObjectTaggingRequest)9 GetObjectTaggingResult (com.amazonaws.services.s3.model.GetObjectTaggingResult)9 Utils (com.vmware.xenon.common.Utils)9 Set (java.util.Set)9 File (java.io.File)8 CreateTagsRequest (com.amazonaws.services.ec2.model.CreateTagsRequest)7 Reservation (com.amazonaws.services.ec2.model.Reservation)7 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)6 TagState (com.vmware.photon.controller.model.resources.TagService.TagState)6 DeferredResult (com.vmware.xenon.common.DeferredResult)6