use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.
the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionStoragePolicyFilterStorageFileExistenceValidationNotEnabled.
@Test
public void testInitiateStoragePolicyTransitionStoragePolicyFilterStorageFileExistenceValidationNotEnabled() {
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
// Create and persist a business object data entity.
businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create an S3 storage without the S3 file existence validation enabled.
List<Attribute> attributes = new ArrayList<>();
attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME));
attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), Boolean.FALSE.toString()));
attributes.add(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), Boolean.TRUE.toString()));
storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, attributes);
// Create and persist a storage policy entity with storage policy filter storage has no S3 bucket name attribute configured.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Try to initiate a storage policy transition when storage policy filter storage has no S3 file existence validation enabled.
try {
storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
fail("Should throw an IllegalStateException when storage policy filter storage has no S3 file existence validation enabled.");
} catch (IllegalStateException e) {
assertEquals(String.format("File existence validation must be enabled on \"%s\" storage. Storage policy: {%s}", STORAGE_NAME, storagePolicyServiceTestHelper.getExpectedStoragePolicyKeyAndVersionAsString(storagePolicyKey, INITIAL_VERSION)), e.getMessage());
}
}
use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.
the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionSubpartitionValuesWithoutFormatSchema.
@Test
public void testInitiateStoragePolicyTransitionSubpartitionValuesWithoutFormatSchema() throws Exception {
// Create and persist the relative database entities.
storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
// Create a business object data key. Please note that business object data key contains
// sub-partition value that would require the relative business object format to have schema.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
// Create and persist a storage unit. Please note that the business object format is not going to have a schema.
storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create and persist a storage policy entity.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Override configuration to specify some settings required for testing.
Map<String, Object> overrideMap = new HashMap<>();
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
modifyPropertySourceInEnvironment(overrideMap);
// Try to initiate a storage policy transition when sub-partition values are used and format has no schema.
try {
storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
fail("Should throw an IllegalArgumentException when sub-partition values are used and format has no schema.");
} catch (IllegalArgumentException e) {
assertEquals(String.format("Schema must be defined when using subpartition values for business object format {%s}.", businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatKeyAsString(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION)), e.getMessage());
} finally {
// Restore the property sources so we don't affect other tests.
restorePropertySourceInEnvironment();
}
}
use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.
the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionStorageUnitInInvalidState.
@Test
public void testInitiateStoragePolicyTransitionStorageUnitInInvalidState() throws Exception {
// Create and persist the relative database entities.
storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
// Create and persist a storage unit with an invalid storage unit status.
storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, STORAGE_UNIT_STATUS, NO_STORAGE_DIRECTORY_PATH);
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create and persist a storage policy entity.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Override configuration to specify some settings required for testing.
Map<String, Object> overrideMap = new HashMap<>();
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
modifyPropertySourceInEnvironment(overrideMap);
// Try to initiate a storage policy transition when storage unit has an invalid status.
try {
storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
fail();
} catch (IllegalArgumentException e) {
assertEquals(String.format("Storage unit status is \"%s\", but must be \"%s\" or \"%s\" for storage policy transition to proceed. " + "Storage: {%s}, business object data: {%s}", STORAGE_UNIT_STATUS, StorageUnitStatusEntity.ENABLED, StorageUnitStatusEntity.ARCHIVING, STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
} finally {
// Restore the property sources so we don't affect other tests.
restorePropertySourceInEnvironment();
}
}
use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.
the class StoragePolicySelectorServiceTest method testExecutePrimaryPartitionValueStoragePolicyRuleType.
@Test
public void testExecutePrimaryPartitionValueStoragePolicyRuleType() throws Exception {
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create and persist a storage policy entity.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_PRIMARY_PARTITION_VALUE, BDATA_PARTITION_VALUE_AGE_IN_DAYS, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Create a partition value that would satisfy the primary partition value age check as per the storage policy rule.
String primaryPartitionValue = getTestPrimaryPartitionValue(BDATA_PARTITION_VALUE_AGE_IN_DAYS + 1);
// Create and persist a storage unit in the storage policy filter storage.
storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
// Overwrite the "updated on" threshold for a newly created business object data to be selectable by the storage policy.
Map<String, Object> overrideMap = new HashMap<>();
overrideMap.put(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS.getKey(), -1);
modifyPropertySourceInEnvironment(overrideMap);
try {
// Execute the storage policy selection.
List<StoragePolicySelection> resultStoragePolicySelections = storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT);
// Validate the results.
assertEquals(Arrays.asList(new StoragePolicySelection(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION), storagePolicyKey, INITIAL_VERSION)), resultStoragePolicySelections);
} finally {
restorePropertySourceInEnvironment();
}
}
use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.
the class StoragePolicySelectorServiceTest method testExecutePrimaryPartitionValueStoragePolicyRuleTypeBusinessObjectDataUpdatedTooRecently.
@Test
public void testExecutePrimaryPartitionValueStoragePolicyRuleTypeBusinessObjectDataUpdatedTooRecently() throws Exception {
// Create a storage policy key.
StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
// Create and persist a storage policy entity.
storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_PRIMARY_PARTITION_VALUE, BDATA_PARTITION_VALUE_AGE_IN_DAYS, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
// Create a partition value that would satisfy the primary partition value age check as per the storage policy rule.
String primaryPartitionValue = getTestPrimaryPartitionValue(BDATA_PARTITION_VALUE_AGE_IN_DAYS + 1);
// Create and persist a storage unit in the storage policy filter storage.
storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
// Execute the storage policy selection and validate the results. Since the default value for "updated on"
// threshold is greater than 1 day, no newly created business object data instances can match the storage policy.
assertEquals(0, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
// Overwrite the "updated on" threshold for a newly created business object data to be selectable by the storage policy.
Map<String, Object> overrideMap = new HashMap<>();
overrideMap.put(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS.getKey(), -1);
modifyPropertySourceInEnvironment(overrideMap);
try {
// Execute the storage policy selection and validate the results. One business object data matching to storage policy should get selected.
assertEquals(Arrays.asList(new StoragePolicySelection(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION), storagePolicyKey, INITIAL_VERSION)), storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT));
} finally {
restorePropertySourceInEnvironment();
}
}
Aggregations