use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataStorageFileServiceImpl method createBusinessObjectDataStorageFilesImpl.
/**
* Adds files to Business object data storage.
*
* @param businessObjectDataStorageFilesCreateRequest the business object data storage files create request
*
* @return BusinessObjectDataStorageFilesCreateResponse
*/
protected BusinessObjectDataStorageFilesCreateResponse createBusinessObjectDataStorageFilesImpl(BusinessObjectDataStorageFilesCreateRequest businessObjectDataStorageFilesCreateRequest) {
// validate request
validateBusinessObjectDataStorageFilesCreateRequest(businessObjectDataStorageFilesCreateRequest);
// retrieve and validate that the business object data exists
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(getBusinessObjectDataKey(businessObjectDataStorageFilesCreateRequest));
// Validate that business object data is in one of the pre-registered states.
Assert.isTrue(BooleanUtils.isTrue(businessObjectDataEntity.getStatus().getPreRegistrationStatus()), String.format("Business object data status must be one of the pre-registration statuses. Business object data status {%s}, business object data {%s}", businessObjectDataEntity.getStatus().getCode(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
// retrieve and validate that the storage unit exists
StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(businessObjectDataStorageFilesCreateRequest.getStorageName(), businessObjectDataEntity);
// Validate the storage unit has an acceptable status for adding new files.
Assert.isTrue(StorageUnitStatusEntity.ENABLED.equals(storageUnitEntity.getStatus().getCode()), String.format("Storage unit must be in the ENABLED status. Storage unit status {%s}, business object data {%s}", storageUnitEntity.getStatus().getCode(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity)));
StorageEntity storageEntity = storageUnitEntity.getStorage();
// Get the S3 validation flags.
boolean validatePathPrefix = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX), storageEntity, false, true);
boolean validateFileExistence = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE), storageEntity, false, true);
boolean validateFileSize = storageHelper.getBooleanStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_SIZE), storageEntity, false, true);
// Ensure that file size validation is not enabled without file existence validation.
if (validateFileSize) {
Assert.isTrue(validateFileExistence, String.format("Storage \"%s\" has file size validation enabled without file existence validation.", storageEntity.getName()));
}
// Process the add storage files request based on the auto-discovery of storage files being enabled or not.
List<StorageFile> storageFiles;
if (BooleanUtils.isTrue(businessObjectDataStorageFilesCreateRequest.isDiscoverStorageFiles())) {
// Discover new storage files for this storage unit.
storageFiles = discoverStorageFiles(storageUnitEntity);
} else {
// Get the list of storage files from the request.
storageFiles = businessObjectDataStorageFilesCreateRequest.getStorageFiles();
// Validate storage files.
validateStorageFiles(storageFiles, storageUnitEntity, validatePathPrefix, validateFileExistence, validateFileSize);
}
// Add new storage files to the storage unit.
storageFileDaoHelper.createStorageFileEntitiesFromStorageFiles(storageUnitEntity, storageFiles);
// Construct and return the response.
return createBusinessObjectDataStorageFilesCreateResponse(storageEntity, businessObjectDataEntity, storageFiles);
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class UploadDownloadServiceImpl method initiateDownloadSingle.
@NamespacePermission(fields = "#namespace", permissions = NamespacePermissionEnum.READ)
@Override
public DownloadSingleInitiationResponse initiateDownloadSingle(String namespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, Integer businessObjectFormatVersion, String partitionValue, Integer businessObjectDataVersion) {
// Create the business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(namespace, businessObjectDefinitionName, businessObjectFormatUsage, businessObjectFormatFileType, businessObjectFormatVersion, partitionValue, null, businessObjectDataVersion);
// Validate the parameters
businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
// Retrieve the persisted business object data
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
// Make sure the status of the business object data is VALID
businessObjectDataHelper.assertBusinessObjectDataStatusEquals(BusinessObjectDataStatusEntity.VALID, businessObjectDataEntity);
// Get the external storage registered against this data
// Validate that the storage unit exists
StorageUnitEntity storageUnitEntity = IterableUtils.get(businessObjectDataEntity.getStorageUnits(), 0);
// Validate that the storage unit contains only 1 file
assertHasOneStorageFile(storageUnitEntity);
String s3BucketName = storageHelper.getStorageBucketName(storageUnitEntity.getStorage());
String s3ObjectKey = IterableUtils.get(storageUnitEntity.getStorageFiles(), 0).getPath();
// Get the temporary credentials
Credentials downloaderCredentials = getExternalDownloaderCredentials(storageUnitEntity.getStorage(), String.valueOf(businessObjectDataEntity.getId()), s3ObjectKey);
// Generate a pre-signed URL
Date expiration = downloaderCredentials.getExpiration();
S3FileTransferRequestParamsDto s3BucketAccessParams = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
String presignedUrl = s3Dao.generateGetObjectPresignedUrl(s3BucketName, s3ObjectKey, expiration, s3BucketAccessParams);
// Construct and return the response
DownloadSingleInitiationResponse response = new DownloadSingleInitiationResponse();
response.setBusinessObjectData(businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity));
response.setAwsAccessKey(downloaderCredentials.getAccessKeyId());
response.setAwsSecretKey(downloaderCredentials.getSecretAccessKey());
response.setAwsSessionToken(downloaderCredentials.getSessionToken());
response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(expiration));
response.setPreSignedUrl(presignedUrl);
return response;
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class UploadDownloadServiceImpl method extendUploadSingleCredentials.
@NamespacePermission(fields = "#namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public UploadSingleCredentialExtensionResponse extendUploadSingleCredentials(String namespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, Integer businessObjectFormatVersion, String partitionValue, Integer businessObjectDataVersion) {
// Create the business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(namespace, businessObjectDefinitionName, businessObjectFormatUsage, businessObjectFormatFileType, businessObjectFormatVersion, partitionValue, null, businessObjectDataVersion);
// Validate and trim the business object data key.
businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
// Get the business object data for the key.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
// Ensure the status of the business object data is "uploading" in order to extend credentials.
if (!(businessObjectDataEntity.getStatus().getCode().equals(BusinessObjectDataStatusEntity.UPLOADING))) {
throw new IllegalArgumentException(String.format(String.format("Business object data {%s} has a status of \"%s\" and must be \"%s\" to extend " + "credentials.", businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey), businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING)));
}
// Get the S3 managed "loading dock" storage entity and make sure it exists.
StorageEntity storageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
String s3BucketName = storageHelper.getStorageBucketName(storageEntity);
// Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, businessObjectDataEntity);
// Validate that the storage unit contains exactly one storage file.
assertHasOneStorageFile(storageUnitEntity);
// Get the storage file entity.
StorageFileEntity storageFileEntity = IterableUtils.get(storageUnitEntity.getStorageFiles(), 0);
// Get the storage file path.
String storageFilePath = storageFileEntity.getPath();
String awsRoleArn = getStorageUploadRoleArn(storageEntity);
Integer awsRoleDurationSeconds = getStorageUploadSessionDuration(storageEntity);
String awsKmsKeyId = storageHelper.getStorageKmsKeyId(storageEntity);
// Get the temporary security credentials to access S3_MANAGED_STORAGE.
Credentials assumedSessionCredentials = stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), String.valueOf(businessObjectDataEntity.getId()), awsRoleArn, awsRoleDurationSeconds, createUploaderPolicy(s3BucketName, storageFilePath, awsKmsKeyId));
// Create the response.
UploadSingleCredentialExtensionResponse response = new UploadSingleCredentialExtensionResponse();
response.setAwsAccessKey(assumedSessionCredentials.getAccessKeyId());
response.setAwsSecretKey(assumedSessionCredentials.getSecretAccessKey());
response.setAwsSessionToken(assumedSessionCredentials.getSessionToken());
response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(assumedSessionCredentials.getExpiration()));
return response;
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class StorageUnitServiceImpl method getS3KeyPrefixImpl.
/**
* Gets the S3 key prefix.
*
* @param businessObjectDataKey the business object data key
* @param businessObjectFormatPartitionKey the business object format partition key
* @param storageName the storage name
* @param createNewVersion specifies if it is OK to return an S3 key prefix for a new business object data version that is not an initial version. This
* parameter is ignored, when the business object data version is specified.
*
* @return the S3 key prefix
*/
protected S3KeyPrefixInformation getS3KeyPrefixImpl(BusinessObjectDataKey businessObjectDataKey, String businessObjectFormatPartitionKey, String storageName, Boolean createNewVersion) {
// Validate and trim the business object data key.
businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, false);
// If specified, trim the partition key parameter.
String businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKey;
if (businessObjectFormatPartitionKeyLocal != null) {
businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKeyLocal.trim();
}
// If specified, trim the storage name. Otherwise, default to the configuration option.
String storageNameLocal = storageName;
if (StringUtils.isNotBlank(storageNameLocal)) {
storageNameLocal = storageNameLocal.trim();
} else {
storageNameLocal = configurationHelper.getProperty(ConfigurationValue.S3_STORAGE_NAME_DEFAULT);
}
// Get the business object format for the specified parameters and make sure it exists.
BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoHelper.getBusinessObjectFormatEntity(new BusinessObjectFormatKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion()));
// If specified, ensure that partition key matches what's configured within the business object format.
if (StringUtils.isNotBlank(businessObjectFormatPartitionKeyLocal)) {
Assert.isTrue(businessObjectFormatEntity.getPartitionKey().equalsIgnoreCase(businessObjectFormatPartitionKeyLocal), "Partition key \"" + businessObjectFormatPartitionKeyLocal + "\" doesn't match configured business object format partition key \"" + businessObjectFormatEntity.getPartitionKey() + "\".");
}
// Get and validate the storage along with the relative attributes.
StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageNameLocal);
// If the business object data version is not specified, get the next business object data version value.
if (businessObjectDataKey.getBusinessObjectDataVersion() == null) {
// Get the latest data version for this business object data, if it exists.
BusinessObjectDataEntity latestVersionBusinessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(new BusinessObjectDataKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion(), businessObjectDataKey.getPartitionValue(), businessObjectDataKey.getSubPartitionValues(), null));
// Throw an error if this business object data already exists and createNewVersion flag is not set.
if (latestVersionBusinessObjectDataEntity != null && !createNewVersion) {
throw new AlreadyExistsException("Initial version of the business object data already exists.");
}
businessObjectDataKey.setBusinessObjectDataVersion(latestVersionBusinessObjectDataEntity == null ? BusinessObjectDataEntity.BUSINESS_OBJECT_DATA_INITIAL_VERSION : latestVersionBusinessObjectDataEntity.getVersion() + 1);
}
// Build the S3 key prefix string.
String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey);
// Create and return the S3 key prefix.
S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation();
s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix);
return s3KeyPrefixInformation;
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class UploadDownloadHelperServiceImpl method prepareForFileMoveImpl.
/**
* Prepares to move an S3 file from the source bucket to the target bucket. On success, both the target and source business object data statuses are set to
* "RE-ENCRYPTING" and the DTO is updated accordingly.
*
* @param objectKey the object key (i.e. filename)
* @param completeUploadSingleParamsDto the DTO to be initialized with parameters required for complete upload single message processing
*/
protected void prepareForFileMoveImpl(String objectKey, CompleteUploadSingleParamsDto completeUploadSingleParamsDto) {
try {
// Obtain the source business object data entity.
BusinessObjectDataEntity sourceBusinessObjectDataEntity = storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData();
// Get the status and key of the source business object data entity.
completeUploadSingleParamsDto.setSourceOldStatus(sourceBusinessObjectDataEntity.getStatus().getCode());
completeUploadSingleParamsDto.setSourceBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity));
// Find the target business object data by the source business object data's partition value, which should have been an UUID.
// This is assuming that the target has the same partition value as the source, and that there exist one and only one target
// business object data for this UUID.
BusinessObjectDataEntity targetBusinessObjectDataEntity = getTargetBusinessObjectDataEntity(sourceBusinessObjectDataEntity);
// Get the status and key of the target business object data entity.
completeUploadSingleParamsDto.setTargetOldStatus(targetBusinessObjectDataEntity.getStatus().getCode());
completeUploadSingleParamsDto.setTargetBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity));
// This check effectively discards any duplicate SQS messages coming from S3 for the same uploaded file.
for (BusinessObjectDataEntity businessObjectDataEntity : Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity)) {
if (!BusinessObjectDataStatusEntity.UPLOADING.equals(businessObjectDataEntity.getStatus().getCode())) {
LOGGER.info("Ignoring S3 notification since business object data status \"{}\" does not match the expected status \"{}\". " + "businessObjectDataKey={}", businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING, jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
// method skip the rest of the steps required to complete the upload single message processing.
return;
}
}
// Get the S3 managed "loading dock" storage entity and make sure it exists.
StorageEntity s3ManagedLoadingDockStorageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
// Get bucket name for S3 managed "loading dock" storage. Please note that this attribute value is required.
completeUploadSingleParamsDto.setSourceBucketName(storageHelper.getStorageBucketName(s3ManagedLoadingDockStorageEntity));
// Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
// Get the storage file entity.
StorageFileEntity sourceStorageFileEntity = IterableUtils.get(sourceStorageUnitEntity.getStorageFiles(), 0);
// Get the source storage file path.
completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath());
// Get the AWS parameters.
AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
completeUploadSingleParamsDto.setAwsParams(awsParamsDto);
// Validate the source S3 file.
S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()).withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()).withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build();
s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes());
// Get the S3 managed "external" storage entity and make sure it exists.
StorageEntity s3ManagedExternalStorageEntity = getUniqueStorage(targetBusinessObjectDataEntity);
// Get bucket name for S3 managed "external" storage. Please note that this attribute value is required.
completeUploadSingleParamsDto.setTargetBucketName(storageHelper.getStorageBucketName(s3ManagedExternalStorageEntity));
// Get AWS KMS External Key ID.
completeUploadSingleParamsDto.setKmsKeyId(storageHelper.getStorageKmsKeyId(s3ManagedExternalStorageEntity));
// Make sure the target does not already contain the file.
completeUploadSingleParamsDto.setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath());
assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
try {
// Change the status of the source and target business object data to RE-ENCRYPTING.
businessObjectDataDaoHelper.updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
businessObjectDataDaoHelper.updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
}// caught by a business object data status check that occurs inside the prepareForFileMove() helper method.
catch (OptimisticLockException e) {
LOGGER.info("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey={} targetBusinessObjectDataKey={}", jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), jsonHelper.objectToJson(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
// method skip the rest of the steps required to complete the upload single message processing.
return;
}
// Set new status for the source and target business object data in the DTO.
completeUploadSingleParamsDto.setSourceNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
completeUploadSingleParamsDto.setTargetNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
} catch (RuntimeException e) {
// Update statuses for both the source and target business object data instances.
completeUploadSingleParamsDto.setSourceNewStatus(setAndReturnNewSourceBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()));
// Update statuses for both the source and target business object data instances.
completeUploadSingleParamsDto.setTargetNewStatus(setAndReturnNewTargetBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
// Delete the source S3 file. Please note that the method below only logs runtime exceptions without re-throwing them.
deleteSourceS3ObjectAfterError(completeUploadSingleParamsDto.getSourceBucketName(), completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBusinessObjectDataKey());
// Log the error.
LOGGER.error("Failed to process upload single completion request for file. s3Key=\"{}\"", objectKey, e);
}
// If a status update occurred for the source business object data, create a business object data notification for this event.
if (completeUploadSingleParamsDto.getSourceNewStatus() != null) {
notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getSourceBusinessObjectDataKey(), completeUploadSingleParamsDto.getSourceNewStatus(), completeUploadSingleParamsDto.getSourceOldStatus());
}
// If a status update occurred for the target business object data, create a business object data notification for this event.
if (completeUploadSingleParamsDto.getTargetNewStatus() != null) {
notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getTargetBusinessObjectDataKey(), completeUploadSingleParamsDto.getTargetNewStatus(), completeUploadSingleParamsDto.getTargetOldStatus());
}
}
Aggregations