Examples with StoragePolicyKey org.finra.herd.model.api.xml.StoragePolicyKey used on opensource projects

Example 71 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyServiceImpl method createStoragePolicy.

@NamespacePermissions({ @NamespacePermission(fields = "#request?.storagePolicyKey?.namespace", permissions = NamespacePermissionEnum.WRITE), @NamespacePermission(fields = "#request?.storagePolicyFilter?.namespace", permissions = NamespacePermissionEnum.WRITE) })
@Override
public StoragePolicy createStoragePolicy(StoragePolicyCreateRequest request) {
    // Validate and trim the request parameters.
    validateStoragePolicyCreateRequest(request);
    // Get the storage policy key.
    StoragePolicyKey storagePolicyKey = request.getStoragePolicyKey();
    // Ensure a storage policy with the specified name doesn't already exist for the specified namespace.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDao.getStoragePolicyByAltKey(storagePolicyKey);
    if (storagePolicyEntity != null) {
        throw new AlreadyExistsException(String.format("Unable to create storage policy with name \"%s\" because it already exists for namespace \"%s\".", storagePolicyKey.getStoragePolicyName(), storagePolicyKey.getNamespace()));
    }
    // Retrieve and ensure that namespace exists with the specified storage policy namespace code.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(storagePolicyKey.getNamespace());
    // Retrieve and ensure that storage policy rule type exists.
    StoragePolicyRuleTypeEntity storagePolicyRuleTypeEntity = storagePolicyRuleTypeDaoHelper.getStoragePolicyRuleTypeEntity(request.getStoragePolicyRule().getRuleType());
    // Get the storage policy filter.
    StoragePolicyFilter storagePolicyFilter = request.getStoragePolicyFilter();
    // If specified, retrieve and ensure that the business object definition exists.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity = null;
    if (StringUtils.isNotBlank(storagePolicyFilter.getBusinessObjectDefinitionName())) {
        businessObjectDefinitionEntity = businessObjectDefinitionDaoHelper.getBusinessObjectDefinitionEntity(new BusinessObjectDefinitionKey(storagePolicyFilter.getNamespace(), storagePolicyFilter.getBusinessObjectDefinitionName()));
    }
    // If specified, retrieve and ensure that file type exists.
    FileTypeEntity fileTypeEntity = null;
    if (StringUtils.isNotBlank(storagePolicyFilter.getBusinessObjectFormatFileType())) {
        fileTypeEntity = fileTypeDaoHelper.getFileTypeEntity(storagePolicyFilter.getBusinessObjectFormatFileType());
    }
    // Retrieve and ensure that storage policy filter storage exists.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storagePolicyFilter.getStorageName());
    // Validate the source storage.
    storagePolicyDaoHelper.validateStoragePolicyFilterStorage(storageEntity);
    // Retrieve and ensure that storage policy transition type exists.
    StoragePolicyTransitionTypeEntity storagePolicyTransitionTypeEntity = storagePolicyTransitionTypeDaoHelper.getStoragePolicyTransitionTypeEntity(request.getStoragePolicyTransition().getTransitionType());
    // Retrieve and ensure that specified storage policy status exists.
    StoragePolicyStatusEntity storagePolicyStatusEntity = storagePolicyStatusDaoHelper.getStoragePolicyStatusEntity(request.getStatus());
    // Create and persist a new storage policy entity from the request information.
    storagePolicyEntity = createStoragePolicyEntity(namespaceEntity, storagePolicyKey.getStoragePolicyName(), storageEntity, storagePolicyRuleTypeEntity, request.getStoragePolicyRule().getRuleValue(), businessObjectDefinitionEntity, request.getStoragePolicyFilter().getBusinessObjectFormatUsage(), fileTypeEntity, storagePolicyTransitionTypeEntity, storagePolicyStatusEntity, StoragePolicyEntity.STORAGE_POLICY_INITIAL_VERSION, true);
    // Create and return the storage policy object from the persisted entity.
    return createStoragePolicyFromEntity(storagePolicyEntity);
}

Example 72 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorServiceTest method testProcessStoragePolicySelectionMessageRuntimeException.

@Test
public void testProcessStoragePolicySelectionMessageRuntimeException() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create S3FileTransferRequestParamsDto to access the S3 bucket location.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist a storage unit in the storage.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Get the storage files.
    List<StorageFile> storageFiles = storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles());
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Put relative S3 files into the S3 bucket, except for the last one.  This should trigger a RuntimeException.
        for (StorageFile storageFile : storageFiles.subList(0, storageFiles.size() - 1)) {
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[storageFile.getFileSizeBytes().intValue()]), null), null);
        }
        // Try to perform a storage policy transition.
        storagePolicyProcessorService.processStoragePolicySelectionMessage(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        fail();
    } catch (RuntimeException e) {
        // Validate the exception error message.
        assertEquals(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", storageFiles.get(storageFiles.size() - 1).getFilePath(), STORAGE_NAME), e.getMessage());
        // Validate the StoragePolicyTransitionFailedAttempts value.
        assertEquals(Integer.valueOf(1), storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
        // Validate the status of the storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVING, storageUnitEntity.getStatus().getCode());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}

Example 73 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionBusinessObjectDataNoExists.

@Test
public void testInitiateStoragePolicyTransitionBusinessObjectDataNoExists() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Try to initiate a storage policy transition when business object data does not exist.
    try {
        storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        fail("Should throw an ObjectNotFoundException when business object data does not exist.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(businessObjectDataKey, null), e.getMessage());
    }
}

Example 74 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionStoragePolicyNoExists.

@Test
public void testInitiateStoragePolicyTransitionStoragePolicyNoExists() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist a business object data entity.
    businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Try to initiate a storage policy transition when storage policy does not exist.
    try {
        storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION));
        fail("Should throw an ObjectNotFoundException when storage policy does not exist.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Storage policy with name \"%s\" and version \"%d\" does not exist for \"%s\" namespace.", storagePolicyKey.getStoragePolicyName(), INITIAL_VERSION, storagePolicyKey.getNamespace()), e.getMessage());
    }
}

Example 75 with StoragePolicyKey

use of org.finra.herd.model.api.xml.StoragePolicyKey in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceTest method testInitiateStoragePolicyTransitionOtherBusinessObjectDataHasStorageFilesMatchingS3KeyPrefix.

@Test
public void testInitiateStoragePolicyTransitionOtherBusinessObjectDataHasStorageFilesMatchingS3KeyPrefix() throws Exception {
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create two business object data keys.
    List<BusinessObjectDataKey> businessObjectDataKeys = Arrays.asList(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION), new BusinessObjectDataKey(BDEF_NAMESPACE_2, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION));
    // For the business object data keys, create and persist two storage units.
    List<StorageUnitEntity> storageUnitEntities = Arrays.asList(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKeys.get(0), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH), storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKeys.get(1), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH));
    // Get the expected S3 key prefix for the business object data key.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // To both storage unit add storage files having the same S3 key prefix.
    List<StorageFileEntity> storageFileEntities = Arrays.asList(storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntities.get(0), expectedS3KeyPrefix + "/" + LOCAL_FILES.get(0), FILE_SIZE_1_KB, ROW_COUNT_1000), storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntities.get(1), expectedS3KeyPrefix + "/" + LOCAL_FILES.get(1), FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    // business object data storage files matching the expected S3 key prefix.
    try {
        storagePolicyProcessorHelperService.initiateStoragePolicyTransition(new StoragePolicyTransitionParamsDto(), new StoragePolicySelection(businessObjectDataKeys.get(0), storagePolicyKey, INITIAL_VERSION));
        fail();
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found %d registered storage file(s) matching business object data S3 key prefix in the storage that is not equal to the number " + "of storage files (%d) registered with the business object data in that storage. " + "Storage: {%s}, s3KeyPrefix {%s}, business object data: {%s}", storageFileEntities.size(), 1, STORAGE_NAME, expectedS3KeyPrefix + "/", businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKeys.get(0))), e.getMessage());
    } finally {
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}