use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.
the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageBusinessObjectDataNoExists.
@Test
public void testProcessMessageBusinessObjectDataNoExists() throws Exception {
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Perform a storage policy transition.
executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
});
}
use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.
the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageStorageUnitAlreadyArchived.
@Test
public void testProcessMessageStorageUnitAlreadyArchived() throws Exception {
// Build the expected S3 key prefix for test business object data.
String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
// Create and persist the relative database entities.
storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
// Create and persist an already ARCHIVED storage unit in the source storage.
StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ARCHIVED, NO_STORAGE_DIRECTORY_PATH);
// Add storage files to the source storage unit.
for (String filePath : LOCAL_FILES) {
storageFileDaoTestHelper.createStorageFileEntity(sourceStorageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
}
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create and persist a storage policy entity.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Override configuration to specify some settings required for testing.
Map<String, Object> overrideMap = new HashMap<>();
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
modifyPropertySourceInEnvironment(overrideMap);
try {
// Try to perform a storage policy transition.
executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
});
// Validate the status of the source storage unit.
assertEquals(StorageUnitStatusEntity.ARCHIVED, sourceStorageUnitEntity.getStatus().getCode());
} finally {
// Restore the property sources so we don't affect other tests.
restorePropertySourceInEnvironment();
}
}
use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.
the class StoragePolicyServiceImpl method createStoragePolicyFromEntity.
/**
* Creates the storage policy registration from the persisted entity.
*
* @param storagePolicyEntity the storage policy registration entity
*
* @return the storage policy registration
*/
private StoragePolicy createStoragePolicyFromEntity(StoragePolicyEntity storagePolicyEntity) {
StoragePolicy storagePolicy = new StoragePolicy();
storagePolicy.setId(storagePolicyEntity.getId());
StoragePolicyKey storagePolicyKey = new StoragePolicyKey();
storagePolicy.setStoragePolicyKey(storagePolicyKey);
storagePolicyKey.setNamespace(storagePolicyEntity.getNamespace().getCode());
storagePolicyKey.setStoragePolicyName(storagePolicyEntity.getName());
StoragePolicyRule storagePolicyRule = new StoragePolicyRule();
storagePolicy.setStoragePolicyRule(storagePolicyRule);
storagePolicyRule.setRuleType(storagePolicyEntity.getStoragePolicyRuleType().getCode());
storagePolicyRule.setRuleValue(storagePolicyEntity.getStoragePolicyRuleValue());
StoragePolicyFilter storagePolicyFilter = new StoragePolicyFilter();
storagePolicy.setStoragePolicyFilter(storagePolicyFilter);
storagePolicyFilter.setNamespace(storagePolicyEntity.getBusinessObjectDefinition() != null ? storagePolicyEntity.getBusinessObjectDefinition().getNamespace().getCode() : null);
storagePolicyFilter.setBusinessObjectDefinitionName(storagePolicyEntity.getBusinessObjectDefinition() != null ? storagePolicyEntity.getBusinessObjectDefinition().getName() : null);
storagePolicyFilter.setBusinessObjectFormatUsage(storagePolicyEntity.getUsage());
storagePolicyFilter.setBusinessObjectFormatFileType(storagePolicyEntity.getFileType() != null ? storagePolicyEntity.getFileType().getCode() : null);
storagePolicyFilter.setStorageName(storagePolicyEntity.getStorage().getName());
StoragePolicyTransition storagePolicyTransition = new StoragePolicyTransition();
storagePolicy.setStoragePolicyTransition(storagePolicyTransition);
storagePolicyTransition.setTransitionType(storagePolicyEntity.getStoragePolicyTransitionType().getCode());
storagePolicy.setStatus(storagePolicyEntity.getStatus().getCode());
return storagePolicy;
}
use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.
the class StoragePolicyProcessorHelperServiceImpl method initiateStoragePolicyTransitionImpl.
/**
* Initiates a storage policy transition as per specified storage policy selection. This method also updates storage policy transition DTO with parameters
* needed to perform a storage policy transition.
*
* @param storagePolicyTransitionParamsDto the storage policy transition DTO to be updated with parameters needed to perform a storage policy transition
* @param storagePolicySelection the storage policy selection message
*/
protected void initiateStoragePolicyTransitionImpl(StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto, StoragePolicySelection storagePolicySelection) {
// Validate and trim the storage policy selection message content.
validateStoragePolicySelection(storagePolicySelection);
// Get the business object data and storage policy keys from the storage policy selection message.
BusinessObjectDataKey businessObjectDataKey = storagePolicySelection.getBusinessObjectDataKey();
StoragePolicyKey storagePolicyKey = storagePolicySelection.getStoragePolicyKey();
Integer storagePolicyVersion = storagePolicySelection.getStoragePolicyVersion();
// Retrieve the business object data entity and ensure it exists.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
// Retrieve the storage policy and ensure it exists.
StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, storagePolicyVersion);
// Get the storage name.
String storageName = storagePolicyEntity.getStorage().getName();
// Initialize the storage policy transition parameters DTO by setting business object data key and storage name.
storagePolicyTransitionParamsDto.setBusinessObjectDataKey(businessObjectDataKey);
storagePolicyTransitionParamsDto.setStorageName(storageName);
// Validate the business object data.
validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
// Validate the storage.
validateStorage(storagePolicyEntity.getStorage(), storagePolicyKey, storagePolicyVersion);
// Validate that storage policy filter storage has S3 bucket name configured.
// Please note that since S3 bucket name attribute value is required we pass a "true" flag.
String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storagePolicyEntity.getStorage(), true);
// Validate that storage policy transition type is GLACIER.
Assert.isTrue(StoragePolicyTransitionTypeEntity.GLACIER.equals(storagePolicyEntity.getStoragePolicyTransitionType().getCode()), String.format("Storage policy transition type \"%s\" is not supported. Storage policy: {%s}", storagePolicyEntity.getStoragePolicyTransitionType().getCode(), storagePolicyHelper.storagePolicyKeyAndVersionToString(storagePolicyKey, storagePolicyVersion)));
// Get the S3 object tag key to be used to tag the objects for archiving.
String s3ObjectTagKey = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY);
// Get the S3 object tag value to be used to tag S3 objects for archiving to Glacier.
String s3ObjectTagValue = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE);
// Get the ARN of the role to assume to tag S3 objects for archiving to Glacier.
String s3ObjectTaggerRoleArn = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN);
// Get the session identifier for the assumed role to be used to tag S3 objects for archiving to Glacier.
String s3ObjectTaggerRoleSessionName = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME);
// Retrieve the storage unit and ensure it exists.
StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(storageName, businessObjectDataEntity);
// Validate the storage unit.
validateStorageUnit(storageUnitEntity, storageName, businessObjectDataKey);
// Get S3 key prefix for this business object data.
String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
// Retrieve and validate storage files registered with the storage unit.
List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
// Validate that this storage does not have any other registered storage files that
// start with the S3 key prefix, but belong to other business object data instances.
storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
// Update the storage unit status.
String reason = StorageUnitStatusEntity.ARCHIVING;
String oldStorageUnitStatus = storageUnitEntity.getStatus().getCode();
storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, reason);
// Update the policy transition parameters DTO.
storagePolicyTransitionParamsDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
storagePolicyTransitionParamsDto.setS3BucketName(s3BucketName);
storagePolicyTransitionParamsDto.setS3KeyPrefix(s3KeyPrefix);
storagePolicyTransitionParamsDto.setNewStorageUnitStatus(storageUnitEntity.getStatus().getCode());
storagePolicyTransitionParamsDto.setOldStorageUnitStatus(oldStorageUnitStatus);
storagePolicyTransitionParamsDto.setStorageFiles(storageFiles);
storagePolicyTransitionParamsDto.setS3ObjectTagKey(s3ObjectTagKey);
storagePolicyTransitionParamsDto.setS3ObjectTagValue(s3ObjectTagValue);
storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleArn(s3ObjectTaggerRoleArn);
storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleSessionName(s3ObjectTaggerRoleSessionName);
}
use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.
the class StoragePolicyRestControllerTest method testGetStoragePolicy.
@Test
public void testGetStoragePolicy() {
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
StoragePolicy storagePolicy = new StoragePolicy(ID, storagePolicyKey, new StoragePolicyRule(STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE), new StoragePolicyFilter(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME), new StoragePolicyTransition(STORAGE_POLICY_TRANSITION_TYPE), StoragePolicyStatusEntity.ENABLED);
when(storagePolicyService.getStoragePolicy(storagePolicyKey)).thenReturn(storagePolicy);
StoragePolicy resultStoragePolicy = storagePolicyRestController.getStoragePolicy(storagePolicyKey.getNamespace(), storagePolicyKey.getStoragePolicyName());
// Verify the external calls.
verify(storagePolicyService).getStoragePolicy(storagePolicyKey);
verifyNoMoreInteractions(storagePolicyService);
// Validate the returned object.
assertEquals(storagePolicy, resultStoragePolicy);
}
Aggregations