Examples with StoragePolicyKey org.finra.herd.model.api.xml.StoragePolicyKey used on opensource projects

Example 1 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageBusinessObjectDataNoExists.

@Test
public void testProcessMessageBusinessObjectDataNoExists() throws Exception {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Perform a storage policy transition.
    executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
        storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
    });
}

Example 2 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageStorageUnitAlreadyArchived.

@Test
public void testProcessMessageStorageUnitAlreadyArchived() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist an already ARCHIVED storage unit in the source storage.
    StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ARCHIVED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the source storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(sourceStorageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Try to perform a storage policy transition.
        executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
            storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
        });
        // Validate the status of the source storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVED, sourceStorageUnitEntity.getStatus().getCode());
    } finally {
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}

Example 3 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyServiceImpl method createStoragePolicyFromEntity.

/**
 * Creates the storage policy registration from the persisted entity.
 *
 * @param storagePolicyEntity the storage policy registration entity
 *
 * @return the storage policy registration
 */
private StoragePolicy createStoragePolicyFromEntity(StoragePolicyEntity storagePolicyEntity) {
    StoragePolicy storagePolicy = new StoragePolicy();
    storagePolicy.setId(storagePolicyEntity.getId());
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey();
    storagePolicy.setStoragePolicyKey(storagePolicyKey);
    storagePolicyKey.setNamespace(storagePolicyEntity.getNamespace().getCode());
    storagePolicyKey.setStoragePolicyName(storagePolicyEntity.getName());
    StoragePolicyRule storagePolicyRule = new StoragePolicyRule();
    storagePolicy.setStoragePolicyRule(storagePolicyRule);
    storagePolicyRule.setRuleType(storagePolicyEntity.getStoragePolicyRuleType().getCode());
    storagePolicyRule.setRuleValue(storagePolicyEntity.getStoragePolicyRuleValue());
    StoragePolicyFilter storagePolicyFilter = new StoragePolicyFilter();
    storagePolicy.setStoragePolicyFilter(storagePolicyFilter);
    storagePolicyFilter.setNamespace(storagePolicyEntity.getBusinessObjectDefinition() != null ? storagePolicyEntity.getBusinessObjectDefinition().getNamespace().getCode() : null);
    storagePolicyFilter.setBusinessObjectDefinitionName(storagePolicyEntity.getBusinessObjectDefinition() != null ? storagePolicyEntity.getBusinessObjectDefinition().getName() : null);
    storagePolicyFilter.setBusinessObjectFormatUsage(storagePolicyEntity.getUsage());
    storagePolicyFilter.setBusinessObjectFormatFileType(storagePolicyEntity.getFileType() != null ? storagePolicyEntity.getFileType().getCode() : null);
    storagePolicyFilter.setStorageName(storagePolicyEntity.getStorage().getName());
    StoragePolicyTransition storagePolicyTransition = new StoragePolicyTransition();
    storagePolicy.setStoragePolicyTransition(storagePolicyTransition);
    storagePolicyTransition.setTransitionType(storagePolicyEntity.getStoragePolicyTransitionType().getCode());
    storagePolicy.setStatus(storagePolicyEntity.getStatus().getCode());
    return storagePolicy;
}

Example 4 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImpl method initiateStoragePolicyTransitionImpl.

/**
 * Initiates a storage policy transition as per specified storage policy selection. This method also updates storage policy transition DTO with parameters
 * needed to perform a storage policy transition.
 *
 * @param storagePolicyTransitionParamsDto the storage policy transition DTO to be updated with parameters needed to perform a storage policy transition
 * @param storagePolicySelection the storage policy selection message
 */
protected void initiateStoragePolicyTransitionImpl(StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto, StoragePolicySelection storagePolicySelection) {
    // Validate and trim the storage policy selection message content.
    validateStoragePolicySelection(storagePolicySelection);
    // Get the business object data and storage policy keys from the storage policy selection message.
    BusinessObjectDataKey businessObjectDataKey = storagePolicySelection.getBusinessObjectDataKey();
    StoragePolicyKey storagePolicyKey = storagePolicySelection.getStoragePolicyKey();
    Integer storagePolicyVersion = storagePolicySelection.getStoragePolicyVersion();
    // Retrieve the business object data entity and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve the storage policy and ensure it exists.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, storagePolicyVersion);
    // Get the storage name.
    String storageName = storagePolicyEntity.getStorage().getName();
    // Initialize the storage policy transition parameters DTO by setting business object data key and storage name.
    storagePolicyTransitionParamsDto.setBusinessObjectDataKey(businessObjectDataKey);
    storagePolicyTransitionParamsDto.setStorageName(storageName);
    // Validate the business object data.
    validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
    // Validate the storage.
    validateStorage(storagePolicyEntity.getStorage(), storagePolicyKey, storagePolicyVersion);
    // Validate that storage policy filter storage has S3 bucket name configured.
    // Please note that since S3 bucket name attribute value is required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storagePolicyEntity.getStorage(), true);
    // Validate that storage policy transition type is GLACIER.
    Assert.isTrue(StoragePolicyTransitionTypeEntity.GLACIER.equals(storagePolicyEntity.getStoragePolicyTransitionType().getCode()), String.format("Storage policy transition type \"%s\" is not supported. Storage policy: {%s}", storagePolicyEntity.getStoragePolicyTransitionType().getCode(), storagePolicyHelper.storagePolicyKeyAndVersionToString(storagePolicyKey, storagePolicyVersion)));
    // Get the S3 object tag key to be used to tag the objects for archiving.
    String s3ObjectTagKey = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY);
    // Get the S3 object tag value to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTagValue = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE);
    // Get the ARN of the role to assume to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleArn = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN);
    // Get the session identifier for the assumed role to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleSessionName = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME);
    // Retrieve the storage unit and ensure it exists.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(storageName, businessObjectDataEntity);
    // Validate the storage unit.
    validateStorageUnit(storageUnitEntity, storageName, businessObjectDataKey);
    // Get S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve and validate storage files registered with the storage unit.
    List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
    // Validate that this storage does not have any other registered storage files that
    // start with the S3 key prefix, but belong to other business object data instances.
    storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
    // Update the storage unit status.
    String reason = StorageUnitStatusEntity.ARCHIVING;
    String oldStorageUnitStatus = storageUnitEntity.getStatus().getCode();
    storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, reason);
    // Update the policy transition parameters DTO.
    storagePolicyTransitionParamsDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
    storagePolicyTransitionParamsDto.setS3BucketName(s3BucketName);
    storagePolicyTransitionParamsDto.setS3KeyPrefix(s3KeyPrefix);
    storagePolicyTransitionParamsDto.setNewStorageUnitStatus(storageUnitEntity.getStatus().getCode());
    storagePolicyTransitionParamsDto.setOldStorageUnitStatus(oldStorageUnitStatus);
    storagePolicyTransitionParamsDto.setStorageFiles(storageFiles);
    storagePolicyTransitionParamsDto.setS3ObjectTagKey(s3ObjectTagKey);
    storagePolicyTransitionParamsDto.setS3ObjectTagValue(s3ObjectTagValue);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleArn(s3ObjectTaggerRoleArn);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleSessionName(s3ObjectTaggerRoleSessionName);
}

Example 5 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyRestControllerTest method testGetStoragePolicy.

@Test
public void testGetStoragePolicy() {
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    StoragePolicy storagePolicy = new StoragePolicy(ID, storagePolicyKey, new StoragePolicyRule(STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE), new StoragePolicyFilter(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME), new StoragePolicyTransition(STORAGE_POLICY_TRANSITION_TYPE), StoragePolicyStatusEntity.ENABLED);
    when(storagePolicyService.getStoragePolicy(storagePolicyKey)).thenReturn(storagePolicy);
    StoragePolicy resultStoragePolicy = storagePolicyRestController.getStoragePolicy(storagePolicyKey.getNamespace(), storagePolicyKey.getStoragePolicyName());
    // Verify the external calls.
    verify(storagePolicyService).getStoragePolicy(storagePolicyKey);
    verifyNoMoreInteractions(storagePolicyService);
    // Validate the returned object.
    assertEquals(storagePolicy, resultStoragePolicy);
}