Search in sources :

Example 1 with Transition

use of com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Transition in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImplTest method testExecuteStoragePolicyTransitionImpl.

@Test
public void testExecuteStoragePolicyTransitionImpl() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage file path.
    String storageFilePath = TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create a list of storage files to be passed as an input.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a storage policy transition parameters DTO.
    StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto = new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create a list of S3 object summaries selected without zero byte directory markers.
    List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = Arrays.asList(new S3ObjectSummary());
    // Create a list of all S3 files matching the S3 key prefix form the S3 bucket.
    List<S3ObjectSummary> actualS3Files = Arrays.asList(new S3ObjectSummary());
    // Create a list of storage files selected for S3 object tagging.
    List<StorageFile> storageFilesSelectedForTagging = Arrays.asList(new StorageFile());
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForTagging = Arrays.asList(new File(storageFilePath));
    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    updatedS3FileTransferRequestParamsDto.setFiles(filesSelectedForTagging);
    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME)).thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(actualS3FilesWithoutZeroByteDirectoryMarkers);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(actualS3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesSelectedForTagging);
    when(storageFileHelper.getFiles(storageFilesSelectedForTagging)).thenReturn(filesSelectedForTagging);
    // Call the method under test.
    storagePolicyProcessorHelperServiceImpl.executeStoragePolicyTransitionImpl(storagePolicyTransitionParamsDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, actualS3FilesWithoutZeroByteDirectoryMarkers, STORAGE_NAME, businessObjectDataKey);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, false);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
    verify(storageFileHelper).getFiles(storageFilesSelectedForTagging);
    verify(s3Service).tagObjects(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME), storagePolicyTransitionParamsDto);
}
Also used : S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) StorageFile(org.finra.herd.model.api.xml.StorageFile) StoragePolicyTransitionParamsDto(org.finra.herd.model.dto.StoragePolicyTransitionParamsDto) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Tag(com.amazonaws.services.s3.model.Tag) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) File(java.io.File) StorageFile(org.finra.herd.model.api.xml.StorageFile) AbstractServiceTest(org.finra.herd.service.AbstractServiceTest) Test(org.junit.Test)

Example 2 with Transition

use of com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Transition in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImpl method executeStoragePolicyTransitionImpl.

/**
 * Executes a storage policy transition as per specified storage policy selection.
 *
 * @param storagePolicyTransitionParamsDto the storage policy transition DTO that contains parameters needed to perform a storage policy transition
 */
protected void executeStoragePolicyTransitionImpl(StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto) {
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3Endpoint(storagePolicyTransitionParamsDto.getS3Endpoint());
    s3FileTransferRequestParamsDto.setS3BucketName(storagePolicyTransitionParamsDto.getS3BucketName());
    s3FileTransferRequestParamsDto.setS3KeyPrefix(StringUtils.appendIfMissing(storagePolicyTransitionParamsDto.getS3KeyPrefix(), "/"));
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = storageHelper.getS3FileTransferRequestParamsDtoByRole(storagePolicyTransitionParamsDto.getS3ObjectTaggerRoleArn(), storagePolicyTransitionParamsDto.getS3ObjectTaggerRoleSessionName());
    s3ObjectTaggerParamsDto.setS3Endpoint(storagePolicyTransitionParamsDto.getS3Endpoint());
    // Get actual S3 files by selecting all S3 keys matching the S3 key prefix form the S3 bucket.
    // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
    // Validate existence of the S3 files.
    storageFileHelper.validateRegisteredS3Files(storagePolicyTransitionParamsDto.getStorageFiles(), actualS3FilesWithoutZeroByteDirectoryMarkers, storagePolicyTransitionParamsDto.getStorageName(), storagePolicyTransitionParamsDto.getBusinessObjectDataKey());
    // Get actual S3 files by selecting all S3 keys matching the S3 key prefix form the S3 bucket.
    // This time, we do not ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, false);
    // Set the list of files to be tagged for archiving.
    s3FileTransferRequestParamsDto.setFiles(storageFileHelper.getFiles(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)));
    // Tag the S3 objects to initiate the archiving.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, new Tag(storagePolicyTransitionParamsDto.getS3ObjectTagKey(), storagePolicyTransitionParamsDto.getS3ObjectTagValue()));
    // Log a list of files tagged in the S3 bucket.
    if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Successfully tagged files in S3 bucket. s3BucketName=\"{}\" s3KeyCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\"", s3FileTransferRequestParamsDto.getS3BucketName(), actualS3Files.size(), storagePolicyTransitionParamsDto.getS3ObjectTagKey(), storagePolicyTransitionParamsDto.getS3ObjectTagValue());
        for (S3ObjectSummary s3File : actualS3FilesWithoutZeroByteDirectoryMarkers) {
            LOGGER.info("s3Key=\"{}\"", s3File.getKey());
        }
    }
}
Also used : S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Tag(com.amazonaws.services.s3.model.Tag)

Example 3 with Transition

use of com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Transition in project herd by FINRAOS.

the class StoragePolicyProcessorServiceTest method testProcessStoragePolicySelectionMessage.

@Test
public void testProcessStoragePolicySelectionMessage() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create S3FileTransferRequestParamsDto to access the S3 bucket location.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist a storage unit in the storage.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Get the storage files.
    List<StorageFile> storageFiles = storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles());
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Put relative S3 files into the S3 bucket.
        for (StorageFile storageFile : storageFiles) {
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[storageFile.getFileSizeBytes().intValue()]), null), null);
        }
        // Perform a storage policy transition.
        storagePolicyProcessorService.processStoragePolicySelectionMessage(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        // Validate the status of the storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVED, storageUnitEntity.getStatus().getCode());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}
Also used : S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) HashMap(java.util.HashMap) StoragePolicySelection(org.finra.herd.model.dto.StoragePolicySelection) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) ByteArrayInputStream(java.io.ByteArrayInputStream) StoragePolicyKey(org.finra.herd.model.api.xml.StoragePolicyKey) StorageFile(org.finra.herd.model.api.xml.StorageFile) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) Test(org.junit.Test)

Example 4 with Transition

use of com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Transition in project herd by FINRAOS.

the class StoragePolicyProcessorServiceTest method testProcessStoragePolicySelectionMessageRuntimeException.

@Test
public void testProcessStoragePolicySelectionMessageRuntimeException() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create S3FileTransferRequestParamsDto to access the S3 bucket location.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist a storage unit in the storage.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Get the storage files.
    List<StorageFile> storageFiles = storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles());
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Put relative S3 files into the S3 bucket, except for the last one.  This should trigger a RuntimeException.
        for (StorageFile storageFile : storageFiles.subList(0, storageFiles.size() - 1)) {
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[storageFile.getFileSizeBytes().intValue()]), null), null);
        }
        // Try to perform a storage policy transition.
        storagePolicyProcessorService.processStoragePolicySelectionMessage(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        fail();
    } catch (RuntimeException e) {
        // Validate the exception error message.
        assertEquals(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", storageFiles.get(storageFiles.size() - 1).getFilePath(), STORAGE_NAME), e.getMessage());
        // Validate the StoragePolicyTransitionFailedAttempts value.
        assertEquals(Integer.valueOf(1), storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
        // Validate the status of the storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVING, storageUnitEntity.getStatus().getCode());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}
Also used : S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) HashMap(java.util.HashMap) StoragePolicySelection(org.finra.herd.model.dto.StoragePolicySelection) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) ByteArrayInputStream(java.io.ByteArrayInputStream) StoragePolicyKey(org.finra.herd.model.api.xml.StoragePolicyKey) StorageFile(org.finra.herd.model.api.xml.StorageFile) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) Test(org.junit.Test)

Example 5 with Transition

use of com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Transition in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method runExecuteStoragePolicyTransitionTest.

private void runExecuteStoragePolicyTransitionTest() {
    // Create S3FileTransferRequestParamsDto to access the S3 bucket location.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build();
    // Create a list of storage files.
    List<StorageFile> storageFiles = new ArrayList<>();
    for (String file : LOCAL_FILES) {
        storageFiles.add(new StorageFile(String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file)), FILE_SIZE_1_KB, ROW_COUNT));
    }
    try {
        // Put relative S3 files into the S3 bucket.
        for (StorageFile storageFile : storageFiles) {
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null), null);
        }
        // Execute a storage policy transition.
        storagePolicyProcessorHelperService.executeStoragePolicyTransition(new StoragePolicyTransitionParamsDto(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION), STORAGE_NAME, NO_S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, storageFiles, S3_ARCHIVE_TO_GLACIER_TAG_KEY, S3_ARCHIVE_TO_GLACIER_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME));
        // Validate that all S3 files are now tagged.
        for (StorageFile storageFile : storageFiles) {
            GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, storageFile.getFilePath()), null);
            assertEquals(Arrays.asList(new Tag(S3_ARCHIVE_TO_GLACIER_TAG_KEY, S3_ARCHIVE_TO_GLACIER_TAG_VALUE)), getObjectTaggingResult.getTagSet());
        }
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
    }
}
Also used : GetObjectTaggingRequest(com.amazonaws.services.s3.model.GetObjectTaggingRequest) S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) ByteArrayInputStream(java.io.ByteArrayInputStream) StorageFile(org.finra.herd.model.api.xml.StorageFile) ArrayList(java.util.ArrayList) StoragePolicyTransitionParamsDto(org.finra.herd.model.dto.StoragePolicyTransitionParamsDto) Tag(com.amazonaws.services.s3.model.Tag) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) GetObjectTaggingResult(com.amazonaws.services.s3.model.GetObjectTaggingResult)

Aggregations

S3FileTransferRequestParamsDto (org.finra.herd.model.dto.S3FileTransferRequestParamsDto)6 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)5 StorageFile (org.finra.herd.model.api.xml.StorageFile)5 PutObjectRequest (com.amazonaws.services.s3.model.PutObjectRequest)4 Tag (com.amazonaws.services.s3.model.Tag)4 ByteArrayInputStream (java.io.ByteArrayInputStream)4 Test (org.junit.Test)4 HashMap (java.util.HashMap)3 StoragePolicyKey (org.finra.herd.model.api.xml.StoragePolicyKey)3 StoragePolicySelection (org.finra.herd.model.dto.StoragePolicySelection)3 StorageUnitEntity (org.finra.herd.model.jpa.StorageUnitEntity)3 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)2 StoragePolicyTransitionParamsDto (org.finra.herd.model.dto.StoragePolicyTransitionParamsDto)2 AbstractServiceTest (org.finra.herd.service.AbstractServiceTest)2 AmazonServiceException (com.amazonaws.AmazonServiceException)1 SdkClientException (com.amazonaws.SdkClientException)1 ProfileCredentialsProvider (com.amazonaws.auth.profile.ProfileCredentialsProvider)1 Regions (com.amazonaws.regions.Regions)1 AmazonS3 (com.amazonaws.services.s3.AmazonS3)1 BucketLifecycleConfiguration (com.amazonaws.services.s3.model.BucketLifecycleConfiguration)1