Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 21 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImpl method initiateStoragePolicyTransitionImpl.

/**
 * Initiates a storage policy transition as per specified storage policy selection. This method also updates storage policy transition DTO with parameters
 * needed to perform a storage policy transition.
 *
 * @param storagePolicyTransitionParamsDto the storage policy transition DTO to be updated with parameters needed to perform a storage policy transition
 * @param storagePolicySelection the storage policy selection message
 */
protected void initiateStoragePolicyTransitionImpl(StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto, StoragePolicySelection storagePolicySelection) {
    // Validate and trim the storage policy selection message content.
    validateStoragePolicySelection(storagePolicySelection);
    // Get the business object data and storage policy keys from the storage policy selection message.
    BusinessObjectDataKey businessObjectDataKey = storagePolicySelection.getBusinessObjectDataKey();
    StoragePolicyKey storagePolicyKey = storagePolicySelection.getStoragePolicyKey();
    Integer storagePolicyVersion = storagePolicySelection.getStoragePolicyVersion();
    // Retrieve the business object data entity and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve the storage policy and ensure it exists.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoHelper.getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, storagePolicyVersion);
    // Get the storage name.
    String storageName = storagePolicyEntity.getStorage().getName();
    // Initialize the storage policy transition parameters DTO by setting business object data key and storage name.
    storagePolicyTransitionParamsDto.setBusinessObjectDataKey(businessObjectDataKey);
    storagePolicyTransitionParamsDto.setStorageName(storageName);
    // Validate the business object data.
    validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
    // Validate the storage.
    validateStorage(storagePolicyEntity.getStorage(), storagePolicyKey, storagePolicyVersion);
    // Validate that storage policy filter storage has S3 bucket name configured.
    // Please note that since S3 bucket name attribute value is required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storagePolicyEntity.getStorage(), true);
    // Validate that storage policy transition type is GLACIER.
    Assert.isTrue(StoragePolicyTransitionTypeEntity.GLACIER.equals(storagePolicyEntity.getStoragePolicyTransitionType().getCode()), String.format("Storage policy transition type \"%s\" is not supported. Storage policy: {%s}", storagePolicyEntity.getStoragePolicyTransitionType().getCode(), storagePolicyHelper.storagePolicyKeyAndVersionToString(storagePolicyKey, storagePolicyVersion)));
    // Get the S3 object tag key to be used to tag the objects for archiving.
    String s3ObjectTagKey = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY);
    // Get the S3 object tag value to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTagValue = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE);
    // Get the ARN of the role to assume to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleArn = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN);
    // Get the session identifier for the assumed role to be used to tag S3 objects for archiving to Glacier.
    String s3ObjectTaggerRoleSessionName = configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME);
    // Retrieve the storage unit and ensure it exists.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(storageName, businessObjectDataEntity);
    // Validate the storage unit.
    validateStorageUnit(storageUnitEntity, storageName, businessObjectDataKey);
    // Get S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storagePolicyEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve and validate storage files registered with the storage unit.
    List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
    // Validate that this storage does not have any other registered storage files that
    // start with the S3 key prefix, but belong to other business object data instances.
    storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
    // Update the storage unit status.
    String reason = StorageUnitStatusEntity.ARCHIVING;
    String oldStorageUnitStatus = storageUnitEntity.getStatus().getCode();
    storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, reason);
    // Update the policy transition parameters DTO.
    storagePolicyTransitionParamsDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
    storagePolicyTransitionParamsDto.setS3BucketName(s3BucketName);
    storagePolicyTransitionParamsDto.setS3KeyPrefix(s3KeyPrefix);
    storagePolicyTransitionParamsDto.setNewStorageUnitStatus(storageUnitEntity.getStatus().getCode());
    storagePolicyTransitionParamsDto.setOldStorageUnitStatus(oldStorageUnitStatus);
    storagePolicyTransitionParamsDto.setStorageFiles(storageFiles);
    storagePolicyTransitionParamsDto.setS3ObjectTagKey(s3ObjectTagKey);
    storagePolicyTransitionParamsDto.setS3ObjectTagValue(s3ObjectTagValue);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleArn(s3ObjectTaggerRoleArn);
    storagePolicyTransitionParamsDto.setS3ObjectTaggerRoleSessionName(s3ObjectTaggerRoleSessionName);
}

Example 22 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceImpl method prepareToFinalizeRestoreImpl.

/**
 * Prepares for the business object data finalize restore by validating the S3 storage unit along with other related database entities. The method also
 * creates and returns a business object data restore DTO.
 *
 * @param storageUnitKey the storage unit key
 *
 * @return the DTO that holds various parameters needed to perform a business object data restore
 */
protected BusinessObjectDataRestoreDto prepareToFinalizeRestoreImpl(BusinessObjectDataStorageUnitKey storageUnitKey) {
    // Get the storage name.
    String storageName = storageUnitKey.getStorageName();
    // Get a business object data key.
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(storageUnitKey);
    // Retrieve the business object data and ensure it exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Retrieve the storage unit and validate it.
    StorageUnitEntity storageUnitEntity = getStorageUnit(storageUnitKey.getStorageName(), businessObjectDataEntity);
    // Validate that S3 storage has S3 bucket name configured. Please note that since S3 bucket name attribute value is required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageUnitEntity.getStorage(), true);
    // Get storage specific S3 key prefix for this business object data.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageUnitEntity.getStorage(), businessObjectDataEntity.getBusinessObjectFormat(), businessObjectDataKey);
    // Retrieve and validate storage files registered with the storage unit.
    List<StorageFile> storageFiles = storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey);
    // Validate that this storage does not have any other registered storage files that
    // start with the S3 key prefix, but belong to other business object data instances.
    storageFileDaoHelper.validateStorageFilesCount(storageName, businessObjectDataKey, s3KeyPrefix, storageFiles.size());
    // Build the business object data restore parameters DTO.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto();
    businessObjectDataRestoreDto.setBusinessObjectDataKey(businessObjectDataKey);
    businessObjectDataRestoreDto.setStorageName(storageName);
    businessObjectDataRestoreDto.setS3Endpoint(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT));
    businessObjectDataRestoreDto.setS3BucketName(s3BucketName);
    businessObjectDataRestoreDto.setS3KeyPrefix(s3KeyPrefix);
    businessObjectDataRestoreDto.setStorageFiles(storageFiles);
    // Return the business object data restore parameters DTO.
    return businessObjectDataRestoreDto;
}

Example 23 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelper method validateStorageUnitS3Files.

/**
 * Validates S3 files per storage unit information.
 *
 * @param storageUnit the storage unit that contains S3 files to be validated
 * @param actualS3Files the list of the actual S3 files
 * @param s3KeyPrefix the S3 key prefix that was prepended to the S3 file paths, when they were uploaded to S3
 */
public void validateStorageUnitS3Files(StorageUnit storageUnit, List<String> actualS3Files, String s3KeyPrefix) {
    // Validate that all files match the expected S3 key prefix and build a list of registered S3 files.
    List<String> registeredS3Files = new ArrayList<>();
    if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) {
        for (StorageFile storageFile : storageUnit.getStorageFiles()) {
            Assert.isTrue(storageFile.getFilePath().startsWith(s3KeyPrefix), String.format("Storage file S3 key prefix \"%s\" does not match the expected S3 key prefix \"%s\".", storageFile.getFilePath(), s3KeyPrefix));
            registeredS3Files.add(storageFile.getFilePath());
        }
    }
    // Validate that all files exist in S3 managed bucket.
    if (!actualS3Files.containsAll(registeredS3Files)) {
        registeredS3Files.removeAll(actualS3Files);
        throw new IllegalStateException(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", registeredS3Files.get(0), storageUnit.getStorage().getName()));
    }
    // Validate that no other files in S3 managed bucket have the same S3 key prefix.
    if (!registeredS3Files.containsAll(actualS3Files)) {
        actualS3Files.removeAll(registeredS3Files);
        throw new IllegalStateException(String.format("Found S3 file \"%s\" in \"%s\" storage not registered with this business object data.", actualS3Files.get(0), storageUnit.getStorage().getName()));
    }
}

Example 24 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelper method validateCreateRequestStorageFiles.

/**
 * Validates a list of storage files. This will validate and trim appropriate fields.
 *
 * @param storageFiles the list of storage files
 */
public void validateCreateRequestStorageFiles(List<StorageFile> storageFiles) {
    Set<String> storageFilePathValidationSet = new HashSet<>();
    for (StorageFile storageFile : storageFiles) {
        Assert.hasText(storageFile.getFilePath(), "A file path must be specified.");
        storageFile.setFilePath(storageFile.getFilePath().trim());
        Assert.notNull(storageFile.getFileSizeBytes(), "A file size must be specified.");
        // Ensure row count is not negative.
        if (storageFile.getRowCount() != null) {
            Assert.isTrue(storageFile.getRowCount() >= 0, "File \"" + storageFile.getFilePath() + "\" has a row count which is < 0.");
        }
        // Check for duplicates.
        if (storageFilePathValidationSet.contains(storageFile.getFilePath())) {
            throw new IllegalArgumentException(String.format("Duplicate storage file found: %s", storageFile.getFilePath()));
        }
        storageFilePathValidationSet.add(storageFile.getFilePath());
    }
}

Example 25 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelper method validateS3Files.

/**
 * Validates S3 files per list of expected storage files.
 *
 * @param expectedStorageFiles the list of expected S3 files represented by storage files
 * @param actualS3Files the list of actual S3 files represented by S3 object summaries
 * @param storageName the storage name
 * @param businessObjectDataKey the business object data key
 * @param fileDescription the file description (i.e. "registered" or "copied") to be used in the relative error messages
 */
private void validateS3Files(List<StorageFile> expectedStorageFiles, List<S3ObjectSummary> actualS3Files, String storageName, BusinessObjectDataKey businessObjectDataKey, String fileDescription) {
    // Load all actual S3 files into a map for easy access.
    Map<String, StorageFile> actualFilesMap = getStorageFilesMapFromS3ObjectSummaries(actualS3Files);
    // Validate existence and file size for all expected files.
    for (StorageFile expectedFile : expectedStorageFiles) {
        if (!actualFilesMap.containsKey(expectedFile.getFilePath())) {
            throw new ObjectNotFoundException(String.format("%s file \"%s\" does not exist in \"%s\" storage.", StringUtils.capitalize(fileDescription), expectedFile.getFilePath(), storageName));
        } else {
            // Validate the file size.
            StorageFile actualFile = actualFilesMap.get(expectedFile.getFilePath());
            if (!Objects.equals(actualFile.getFileSizeBytes(), expectedFile.getFileSizeBytes())) {
                throw new IllegalStateException(String.format("Specified file size of %d bytes for %s \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.", expectedFile.getFileSizeBytes(), fileDescription, expectedFile.getFilePath(), storageName, actualFile.getFileSizeBytes()));
            }
        }
    }
    // Get a list of actual S3 file paths.
    List<String> actualFilePaths = new ArrayList<>(actualFilesMap.keySet());
    // Get a list of expected file paths.
    List<String> expectedFilePaths = getFilePathsFromStorageFiles(expectedStorageFiles);
    // Validate that no other files in S3 bucket except for expected files have the same S3 key prefix.
    if (!expectedFilePaths.containsAll(actualFilePaths)) {
        actualFilePaths.removeAll(expectedFilePaths);
        throw new IllegalStateException(String.format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating %s S3 files. Business object data {%s}", actualFilePaths.get(0), storageName, fileDescription, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
    }
}