Examples with StoragePolicyEntity org.finra.herd.model.jpa.StoragePolicyEntity used on opensource projects

Example 1 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class BusinessObjectDataRetryStoragePolicyTransitionHelper method retryStoragePolicyTransition.

/**
 * Executes a retry of the storage policy transition and return the business object data information.
 *
 * @param businessObjectDataKey the business object data key
 * @param request the information needed to retry a storage policy transition
 *
 * @return the business object data information
 */
public BusinessObjectData retryStoragePolicyTransition(BusinessObjectDataKey businessObjectDataKey, BusinessObjectDataRetryStoragePolicyTransitionRequest request) {
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Validate and trim the request.
    validateBusinessObjectDataRetryStoragePolicyTransitionRequest(request);
    // Retrieve the business object data and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve and ensure that a storage policy exists with the specified key.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKey(request.getStoragePolicyKey());
    // Validate that storage policy filter matches this business object data, except for the storage.
    Assert.isTrue((storagePolicyEntity.getBusinessObjectDefinition() == null || storagePolicyEntity.getBusinessObjectDefinition().equals(businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition())) && (StringUtils.isBlank(storagePolicyEntity.getUsage()) || storagePolicyEntity.getUsage().equalsIgnoreCase(businessObjectDataEntity.getBusinessObjectFormat().getUsage())) && (storagePolicyEntity.getFileType() == null || storagePolicyEntity.getFileType().equals(businessObjectDataEntity.getBusinessObjectFormat().getFileType())), String.format("Business object data does not match storage policy filter. " + "Storage policy: {%s}, business object data: {%s}", storagePolicyHelper.storagePolicyKeyAndVersionToString(request.getStoragePolicyKey(), storagePolicyEntity.getVersion()), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
    // Validate the storage policy filter storage.
    storagePolicyDaoHelper.validateStoragePolicyFilterStorage(storagePolicyEntity.getStorage());
    // Retrieve and validate a storage unit for this business object data.
    StorageUnitEntity storageUnitEntity = getStorageUnit(businessObjectDataEntity, storagePolicyEntity.getStorage());
    // Get S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve storage files registered with this business object data in the  storage.
    int storageFilesCount = storageUnitEntity.getStorageFiles().size();
    // Validate that we have storage files registered in the storage.
    Assert.isTrue(storageFilesCount > 0, String.format("Business object data has no storage files registered in \"%s\" storage. Business object data: {%s}", storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
    // Retrieve all registered storage files from the storage that start with the S3 key prefix.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    String s3KeyPrefixWithTrailingSlash = StringUtils.appendIfMissing(s3KeyPrefix, "/");
    Long registeredStorageFilesMatchingS3KeyPrefixCount = storageFileDao.getStorageFileCount(storageUnitEntity.getStorage().getName(), s3KeyPrefixWithTrailingSlash);
    // Sanity check for the S3 key prefix.
    if (registeredStorageFilesMatchingS3KeyPrefixCount.intValue() != storageFilesCount) {
        throw new IllegalArgumentException(String.format("Number of storage files (%d) registered for the business object data in \"%s\" storage is not equal to " + "the number of registered storage files (%d) matching \"%s\" S3 key prefix in the same storage. Business object data: {%s}", storageFilesCount, storageUnitEntity.getStorage().getName(), registeredStorageFilesMatchingS3KeyPrefixCount, s3KeyPrefixWithTrailingSlash, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
    }
    // Get the SQS queue name from the system configuration.
    String sqsQueueName = configurationHelper.getProperty(ConfigurationValue.STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE_NAME);
    // Throw IllegalStateException if SQS queue name is not defined.
    if (StringUtils.isBlank(sqsQueueName)) {
        throw new IllegalStateException(String.format("SQS queue name not found. Ensure the \"%s\" configuration entry is configured.", ConfigurationValue.STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE_NAME.getKey()));
    }
    // Create a storage policy selection.
    StoragePolicySelection storagePolicySelection = new StoragePolicySelection(businessObjectDataKey, storagePolicyHelper.getStoragePolicyKey(storagePolicyEntity), storagePolicyEntity.getVersion());
    // Executes SQS specific steps needed to retry a storage policy transition.
    sendStoragePolicySelectionSqsMessage(sqsQueueName, storagePolicySelection);
    // Create and return the business object data object from the entity.
    return businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity);
}

Example 2 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicySelectorServiceImplTest method testCompleteStoragePolicyTransitionImpl.

@Test
public void testCompleteStoragePolicyTransitionImpl() {
    // Create a current timestamp.
    Timestamp currentTimestamp = new Timestamp(LONG_VALUE);
    // Set a number of maximum results.
    Integer maxResults = 10;
    // Create configuration values required for testing.
    Integer updatedOnThresholdInDays = 90;
    Integer storagePolicyTransitionMaxAllowedAttempts = 3;
    // Create an empty mapping of matched business object data entities.
    Map<BusinessObjectDataEntity, StoragePolicyEntity> noMatchingBusinessObjectDataEntities = new HashMap<>();
    // Create an empty list of storage policy selections.
    List<StoragePolicySelection> storagePolicySelections = new ArrayList<>();
    // Mock the external calls.
    when(herdDao.getCurrentTimestamp()).thenReturn(currentTimestamp);
    when(herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS)).thenReturn(updatedOnThresholdInDays);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(0), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(1), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(2), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(3), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS)).thenReturn(storagePolicyTransitionMaxAllowedAttempts);
    // Call the method under test.
    List<StoragePolicySelection> result = storagePolicySelectorServiceImpl.execute(AWS_SQS_QUEUE_NAME, maxResults);
    // Verify the external calls.
    verify(herdDao).getCurrentTimestamp();
    verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS);
    verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(0), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(1), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(2), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(3), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(storagePolicySelections, result);
}

Example 3 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicyServiceImpl method createStoragePolicyEntity.

/**
 * Creates and persists a new storage policy entity.
 *
 * @param namespaceEntity the namespace entity
 * @param storagePolicyName the storage policy name
 * @param storageEntity the storage entity
 * @param storagePolicyRuleTypeEntity the storage policy rule type entity
 * @param storagePolicyRuleValue the storage policy rule value
 * @param businessObjectDefinitionEntity the business object definition entity
 * @param businessObjectFormatUsage the business object format usage
 * @param fileTypeEntity the file type entity
 * @param storagePolicyTransitionTypeEntity the transition type of the storage policy
 * @param storagePolicyStatusEntity the storage policy status entity
 * @param storagePolicyVersion the storage policy version
 * @param storagePolicyLatestVersion specifies if this storage policy is flagged as latest version or not
 *
 * @return the newly created storage policy entity
 */
private StoragePolicyEntity createStoragePolicyEntity(NamespaceEntity namespaceEntity, String storagePolicyName, StorageEntity storageEntity, StoragePolicyRuleTypeEntity storagePolicyRuleTypeEntity, Integer storagePolicyRuleValue, BusinessObjectDefinitionEntity businessObjectDefinitionEntity, String businessObjectFormatUsage, FileTypeEntity fileTypeEntity, StoragePolicyTransitionTypeEntity storagePolicyTransitionTypeEntity, StoragePolicyStatusEntity storagePolicyStatusEntity, Integer storagePolicyVersion, Boolean storagePolicyLatestVersion) {
    StoragePolicyEntity storagePolicyEntity = new StoragePolicyEntity();
    storagePolicyEntity.setNamespace(namespaceEntity);
    storagePolicyEntity.setName(storagePolicyName);
    storagePolicyEntity.setStorage(storageEntity);
    storagePolicyEntity.setStoragePolicyRuleType(storagePolicyRuleTypeEntity);
    storagePolicyEntity.setStoragePolicyRuleValue(storagePolicyRuleValue);
    storagePolicyEntity.setBusinessObjectDefinition(businessObjectDefinitionEntity);
    if (StringUtils.isNotBlank(businessObjectFormatUsage)) {
        storagePolicyEntity.setUsage(businessObjectFormatUsage);
    }
    storagePolicyEntity.setFileType(fileTypeEntity);
    storagePolicyEntity.setStoragePolicyTransitionType(storagePolicyTransitionTypeEntity);
    storagePolicyEntity.setStatus(storagePolicyStatusEntity);
    storagePolicyEntity.setVersion(storagePolicyVersion);
    storagePolicyEntity.setLatestVersion(storagePolicyLatestVersion);
    return storagePolicyDao.saveAndRefresh(storagePolicyEntity);
}

Example 4 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImpl method initiateStoragePolicyTransitionImpl.

/**
 * Initiates a storage policy transition as per specified storage policy selection. This method also updates storage policy transition DTO with parameters
 * needed to perform a storage policy transition.
 *
 * @param storagePolicyTransitionParamsDto the storage policy transition DTO to be updated with parameters needed to perform a storage policy transition
 * @param storagePolicySelection the storage policy selection message
 */
protected void initiateStoragePolicyTransitionImpl(StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto, StoragePolicySelection storagePolicySelection) {
    // Validate and trim the storage policy selection message content.
    validateStoragePolicySelection(storagePolicySelection);
    // Get the business object data and storage policy keys from the storage policy selection message.
    BusinessObjectDataKey businessObjectDataKey = storagePolicySelection.getBusinessObjectDataKey();
    StoragePolicyKey storagePolicyKey = storagePolicySelection.getStoragePolicyKey();
    Integer storagePolicyVersion = storagePolicySelection.getStoragePolicyVersion();
    // Retrieve the business object data entity and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve the storage policy and ensure it exists.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, storagePolicyVersion);
    // Get the storage name.
    String storageName = storagePolicyEntity.getStorage().getName();
    // Initialize the storage policy transition parameters DTO by setting business object data key and storage name.
    storagePolicyTransitionParamsDto.setBusinessObjectDataKey(businessObjectDataKey);
    storagePolicyTransitionParamsDto.setStorageName(storageName);
    // Validate the business object data.
    validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
    // Validate the storage.
    validateStorage(storagePolicyEntity.getStorage(), storagePolicyKey, storagePolicyVersion);
    // Validate that storage policy filter storage has S3 bucket name configured.
    // Please note that since S3 bucket name attribute value is required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storagePolicyEntity.getStorage(), true);
    // Validate that storage policy transition type is GLACIER.
    Assert.isTrue(StoragePolicyTransitionTypeEntity.GLACIER.equals(storagePolicyEntity.getStoragePolicyTransitionType().getCode()), String.format("Storage policy transition type \"%s\" is not supported. Storage policy: {%s}", storagePolicyEntity.getStoragePolicyTransitionType().getCode(), storagePolicyHelper.storagePolicyKeyAndVersionToString(storagePolicyKey, storagePolicyVersion)));
    // Get the S3 object tag key to be used to tag the objects for archiving.
    String s3ObjectTagKey = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY);
    // Get the S3 object tag value to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTagValue = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE);
    // Get the ARN of the role to assume to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleArn = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN);
    // Get the session identifier for the assumed role to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleSessionName = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME);
    // Retrieve the storage unit and ensure it exists.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(storageName, businessObjectDataEntity);
    // Validate the storage unit.
    validateStorageUnit(storageUnitEntity, storageName, businessObjectDataKey);
    // Get S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve and validate storage files registered with the storage unit.
    List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
    // Validate that this storage does not have any other registered storage files that
    // start with the S3 key prefix, but belong to other business object data instances.
    storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
    // Update the storage unit status.
    String reason = StorageUnitStatusEntity.ARCHIVING;
    String oldStorageUnitStatus = storageUnitEntity.getStatus().getCode();
    storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, reason);
    // Update the policy transition parameters DTO.
    storagePolicyTransitionParamsDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
    storagePolicyTransitionParamsDto.setS3BucketName(s3BucketName);
    storagePolicyTransitionParamsDto.setS3KeyPrefix(s3KeyPrefix);
    storagePolicyTransitionParamsDto.setNewStorageUnitStatus(storageUnitEntity.getStatus().getCode());
    storagePolicyTransitionParamsDto.setOldStorageUnitStatus(oldStorageUnitStatus);
    storagePolicyTransitionParamsDto.setStorageFiles(storageFiles);
    storagePolicyTransitionParamsDto.setS3ObjectTagKey(s3ObjectTagKey);
    storagePolicyTransitionParamsDto.setS3ObjectTagValue(s3ObjectTagValue);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleArn(s3ObjectTaggerRoleArn);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleSessionName(s3ObjectTaggerRoleSessionName);
}

Example 5 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicyServiceTest method testUpdateStoragePolicyMissingOptionalParametersPassedAsWhitespace.

@Test
public void testUpdateStoragePolicyMissingOptionalParametersPassedAsWhitespace() {
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting();
    // Create and persist a storage policy entity.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE_2, STORAGE_POLICY_RULE_VALUE_2, BDEF_NAMESPACE_2, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, STORAGE_NAME_2, STORAGE_POLICY_TRANSITION_TYPE_2, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Update a storage policy without specifying any of the optional parameters (passing them as whitespace characters).
    StoragePolicy resultStoragePolicy = storagePolicyService.updateStoragePolicy(storagePolicyKey, storagePolicyServiceTestHelper.createStoragePolicyUpdateRequest(STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BLANK_TEXT, BLANK_TEXT, BLANK_TEXT, BLANK_TEXT, STORAGE_NAME, STORAGE_POLICY_TRANSITION_TYPE, StoragePolicyStatusEntity.DISABLED));
    // Validate the returned object.
    assertEquals(new StoragePolicy(resultStoragePolicy.getId(), storagePolicyKey, new StoragePolicyRule(STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE), new StoragePolicyFilter(null, null, null, null, STORAGE_NAME), new StoragePolicyTransition(STORAGE_POLICY_TRANSITION_TYPE), StoragePolicyStatusEntity.DISABLED), resultStoragePolicy);
    assertTrue(resultStoragePolicy.getId() > storagePolicyEntity.getId());
}