Examples with StoragePolicySelection org.finra.herd.model.dto.StoragePolicySelection used on opensource projects

Example 1 with StoragePolicySelection

use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.

the class BusinessObjectDataRetryStoragePolicyTransitionHelper method retryStoragePolicyTransition.

/**
 * Executes a retry of the storage policy transition and return the business object data information.
 *
 * @param businessObjectDataKey the business object data key
 * @param request the information needed to retry a storage policy transition
 *
 * @return the business object data information
 */
public BusinessObjectData retryStoragePolicyTransition(BusinessObjectDataKey businessObjectDataKey, BusinessObjectDataRetryStoragePolicyTransitionRequest request) {
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Validate and trim the request.
    validateBusinessObjectDataRetryStoragePolicyTransitionRequest(request);
    // Retrieve the business object data and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve and ensure that a storage policy exists with the specified key.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKey(request.getStoragePolicyKey());
    // Validate that storage policy filter matches this business object data, except for the storage.
    Assert.isTrue((storagePolicyEntity.getBusinessObjectDefinition() == null || storagePolicyEntity.getBusinessObjectDefinition().equals(businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition())) && (StringUtils.isBlank(storagePolicyEntity.getUsage()) || storagePolicyEntity.getUsage().equalsIgnoreCase(businessObjectDataEntity.getBusinessObjectFormat().getUsage())) && (storagePolicyEntity.getFileType() == null || storagePolicyEntity.getFileType().equals(businessObjectDataEntity.getBusinessObjectFormat().getFileType())), String.format("Business object data does not match storage policy filter. " + "Storage policy: {%s}, business object data: {%s}", storagePolicyHelper.storagePolicyKeyAndVersionToString(request.getStoragePolicyKey(), storagePolicyEntity.getVersion()), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
    // Validate the storage policy filter storage.
    storagePolicyDaoHelper.validateStoragePolicyFilterStorage(storagePolicyEntity.getStorage());
    // Retrieve and validate a storage unit for this business object data.
    StorageUnitEntity storageUnitEntity = getStorageUnit(businessObjectDataEntity, storagePolicyEntity.getStorage());
    // Get S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve storage files registered with this business object data in the  storage.
    int storageFilesCount = storageUnitEntity.getStorageFiles().size();
    // Validate that we have storage files registered in the storage.
    Assert.isTrue(storageFilesCount > 0, String.format("Business object data has no storage files registered in \"%s\" storage. Business object data: {%s}", storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
    // Retrieve all registered storage files from the storage that start with the S3 key prefix.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    String s3KeyPrefixWithTrailingSlash = StringUtils.appendIfMissing(s3KeyPrefix, "/");
    Long registeredStorageFilesMatchingS3KeyPrefixCount = storageFileDao.getStorageFileCount(storageUnitEntity.getStorage().getName(), s3KeyPrefixWithTrailingSlash);
    // Sanity check for the S3 key prefix.
    if (registeredStorageFilesMatchingS3KeyPrefixCount.intValue() != storageFilesCount) {
        throw new IllegalArgumentException(String.format("Number of storage files (%d) registered for the business object data in \"%s\" storage is not equal to " + "the number of registered storage files (%d) matching \"%s\" S3 key prefix in the same storage. Business object data: {%s}", storageFilesCount, storageUnitEntity.getStorage().getName(), registeredStorageFilesMatchingS3KeyPrefixCount, s3KeyPrefixWithTrailingSlash, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
    }
    // Get the SQS queue name from the system configuration.
    String sqsQueueName = configurationHelper.getProperty(ConfigurationValue.STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE_NAME);
    // Throw IllegalStateException if SQS queue name is not defined.
    if (StringUtils.isBlank(sqsQueueName)) {
        throw new IllegalStateException(String.format("SQS queue name not found. Ensure the \"%s\" configuration entry is configured.", ConfigurationValue.STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE_NAME.getKey()));
    }
    // Create a storage policy selection.
    StoragePolicySelection storagePolicySelection = new StoragePolicySelection(businessObjectDataKey, storagePolicyHelper.getStoragePolicyKey(storagePolicyEntity), storagePolicyEntity.getVersion());
    // Executes SQS specific steps needed to retry a storage policy transition.
    sendStoragePolicySelectionSqsMessage(sqsQueueName, storagePolicySelection);
    // Create and return the business object data object from the entity.
    return businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity);
}

Example 2 with StoragePolicySelection

use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageBusinessObjectDataNoExists.

@Test
public void testProcessMessageBusinessObjectDataNoExists() throws Exception {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Perform a storage policy transition.
    executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
        storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
    });
}

Example 3 with StoragePolicySelection

use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessageStorageUnitAlreadyArchived.

@Test
public void testProcessMessageStorageUnitAlreadyArchived() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist an already ARCHIVED storage unit in the source storage.
    StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ARCHIVED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the source storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(sourceStorageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Try to perform a storage policy transition.
        executeWithoutLogging(StoragePolicyProcessorJmsMessageListener.class, () -> {
            storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
        });
        // Validate the status of the source storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVED, sourceStorageUnitEntity.getStatus().getCode());
    } finally {
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}

Example 4 with StoragePolicySelection

use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.

the class StoragePolicySelectorServiceImplTest method testCompleteStoragePolicyTransitionImpl.

@Test
public void testCompleteStoragePolicyTransitionImpl() {
    // Create a current timestamp.
    Timestamp currentTimestamp = new Timestamp(LONG_VALUE);
    // Set a number of maximum results.
    Integer maxResults = 10;
    // Create configuration values required for testing.
    Integer updatedOnThresholdInDays = 90;
    Integer storagePolicyTransitionMaxAllowedAttempts = 3;
    // Create an empty mapping of matched business object data entities.
    Map<BusinessObjectDataEntity, StoragePolicyEntity> noMatchingBusinessObjectDataEntities = new HashMap<>();
    // Create an empty list of storage policy selections.
    List<StoragePolicySelection> storagePolicySelections = new ArrayList<>();
    // Mock the external calls.
    when(herdDao.getCurrentTimestamp()).thenReturn(currentTimestamp);
    when(herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS)).thenReturn(updatedOnThresholdInDays);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(0), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(1), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(2), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(3), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults)).thenReturn(noMatchingBusinessObjectDataEntities);
    when(herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS)).thenReturn(storagePolicyTransitionMaxAllowedAttempts);
    // Call the method under test.
    List<StoragePolicySelection> result = storagePolicySelectorServiceImpl.execute(AWS_SQS_QUEUE_NAME, maxResults);
    // Verify the external calls.
    verify(herdDao).getCurrentTimestamp();
    verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS);
    verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(0), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(1), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(2), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesMatchingStoragePolicies(StoragePolicySelectorServiceImpl.STORAGE_POLICY_PRIORITY_LEVELS.get(3), StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, storagePolicyTransitionMaxAllowedAttempts, 0, maxResults);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(storagePolicySelections, result);
}

Example 5 with StoragePolicySelection

use of org.finra.herd.model.dto.StoragePolicySelection in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListener method processMessage.

/**
 * Processes a JMS message.
 *
 * @param payload the message payload
 * @param allHeaders the JMS headers
 */
@JmsListener(id = HerdJmsDestinationResolver.SQS_DESTINATION_STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE, containerFactory = "storagePolicyProcessorJmsListenerContainerFactory", destination = HerdJmsDestinationResolver.SQS_DESTINATION_STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE)
public void processMessage(String payload, @Headers Map<Object, Object> allHeaders) {
    LOGGER.info("Message received from the JMS queue. jmsQueueName=\"{}\" jmsMessageHeaders=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE, allHeaders, payload);
    // Process the message as storage policy selection message.
    try {
        // Process messages coming from the storage policy selector job.
        StoragePolicySelection storagePolicySelection = jsonHelper.unmarshallJsonToObject(StoragePolicySelection.class, payload);
        LOGGER.debug("Received storage policy selection message: businessObjectDataKey={} storagePolicyKey={} storagePolicyVersion={}", jsonHelper.objectToJson(storagePolicySelection.getBusinessObjectDataKey()), jsonHelper.objectToJson(storagePolicySelection.getStoragePolicyKey()), storagePolicySelection.getStoragePolicyVersion());
        // Process the storage policy selection message.
        storagePolicyProcessorService.processStoragePolicySelectionMessage(storagePolicySelection);
    } catch (RuntimeException | IOException e) {
        // Log a warning message if storage unit status is already ARCHIVED. Such error case is typically caused by a duplicate SQS message.
        if (e instanceof IllegalArgumentException && e.getMessage().startsWith(String.format("Storage unit status is \"%s\"", StorageUnitStatusEntity.ARCHIVED))) {
            LOGGER.warn("Failed to process message from the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE, payload, e);
        } else // Otherwise, log an error.
        {
            LOGGER.error("Failed to process message from the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_STORAGE_POLICY_SELECTOR_JOB_SQS_QUEUE, payload, e);
        }
    }
}