Examples with StorageUnitEntity org.finra.herd.model.jpa.StorageUnitEntity used on opensource projects

Example 26 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageUnitStatusServiceImpl method updateBusinessObjectDataStorageUnitStatusImpl.

/**
 * Updates status of a business object data storage unit.
 *
 * @param businessObjectDataStorageUnitKey the business object data storage unit key
 * @param request the business object data storage unit status update request
 *
 * @return the business object data storage unit status update response
 */
protected BusinessObjectDataStorageUnitStatusUpdateResponse updateBusinessObjectDataStorageUnitStatusImpl(BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey, BusinessObjectDataStorageUnitStatusUpdateRequest request) {
    // Validate and trim the business object data storage unit key.
    storageUnitHelper.validateBusinessObjectDataStorageUnitKey(businessObjectDataStorageUnitKey);
    // Validate status.
    Assert.hasText(request.getStatus(), "A business object data storage unit status must be specified.");
    request.setStatus(request.getStatus().trim());
    // Retrieve and ensure that a business object data storage unit exists with the specified key.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntityByKey(businessObjectDataStorageUnitKey);
    // Retrieve and ensure the new storage unit status entity exists.
    StorageUnitStatusEntity newStorageUnitStatusEntity = storageUnitStatusDaoHelper.getStorageUnitStatusEntity(request.getStatus());
    // Save the old storage unit status value.
    String oldStorageUnitStatus = storageUnitEntity.getStatus().getCode();
    // Update the storage unit status.
    storageUnitDaoHelper.updateStorageUnitStatus(storageUnitEntity, newStorageUnitStatusEntity, request.getStatus());
    // Get business object data key from the business object data entity.
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.createBusinessObjectDataKeyFromEntity(storageUnitEntity.getBusinessObjectData());
    // Create and return the business object data storage unit status response object.
    BusinessObjectDataStorageUnitStatusUpdateResponse response = new BusinessObjectDataStorageUnitStatusUpdateResponse();
    response.setBusinessObjectDataStorageUnitKey(storageUnitHelper.createBusinessObjectDataStorageUnitKey(businessObjectDataKey, storageUnitEntity.getStorage().getName()));
    response.setStatus(storageUnitEntity.getStatus().getCode());
    response.setPreviousStatus(oldStorageUnitStatus);
    return response;
}

Example 27 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class UploadDownloadServiceImpl method initiateDownloadSingle.

@NamespacePermission(fields = "#namespace", permissions = NamespacePermissionEnum.READ)
@Override
public DownloadSingleInitiationResponse initiateDownloadSingle(String namespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, Integer businessObjectFormatVersion, String partitionValue, Integer businessObjectDataVersion) {
    // Create the business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(namespace, businessObjectDefinitionName, businessObjectFormatUsage, businessObjectFormatFileType, businessObjectFormatVersion, partitionValue, null, businessObjectDataVersion);
    // Validate the parameters
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Retrieve the persisted business object data
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Make sure the status of the business object data is VALID
    businessObjectDataHelper.assertBusinessObjectDataStatusEquals(BusinessObjectDataStatusEntity.VALID, businessObjectDataEntity);
    // Get the external storage registered against this data
    // Validate that the storage unit exists
    StorageUnitEntity storageUnitEntity = IterableUtils.get(businessObjectDataEntity.getStorageUnits(), 0);
    // Validate that the storage unit contains only 1 file
    assertHasOneStorageFile(storageUnitEntity);
    String s3BucketName = storageHelper.getStorageBucketName(storageUnitEntity.getStorage());
    String s3ObjectKey = IterableUtils.get(storageUnitEntity.getStorageFiles(), 0).getPath();
    // Get the temporary credentials
    Credentials downloaderCredentials = getExternalDownloaderCredentials(storageUnitEntity.getStorage(), String.valueOf(businessObjectDataEntity.getId()), s3ObjectKey);
    // Generate a pre-signed URL
    Date expiration = downloaderCredentials.getExpiration();
    S3FileTransferRequestParamsDto s3BucketAccessParams = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
    String presignedUrl = s3Dao.generateGetObjectPresignedUrl(s3BucketName, s3ObjectKey, expiration, s3BucketAccessParams);
    // Construct and return the response
    DownloadSingleInitiationResponse response = new DownloadSingleInitiationResponse();
    response.setBusinessObjectData(businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity));
    response.setAwsAccessKey(downloaderCredentials.getAccessKeyId());
    response.setAwsSecretKey(downloaderCredentials.getSecretAccessKey());
    response.setAwsSessionToken(downloaderCredentials.getSessionToken());
    response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(expiration));
    response.setPreSignedUrl(presignedUrl);
    return response;
}

Example 28 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class UploadDownloadServiceImpl method extendUploadSingleCredentials.

@NamespacePermission(fields = "#namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public UploadSingleCredentialExtensionResponse extendUploadSingleCredentials(String namespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, Integer businessObjectFormatVersion, String partitionValue, Integer businessObjectDataVersion) {
    // Create the business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(namespace, businessObjectDefinitionName, businessObjectFormatUsage, businessObjectFormatFileType, businessObjectFormatVersion, partitionValue, null, businessObjectDataVersion);
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Get the business object data for the key.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Ensure the status of the business object data is "uploading" in order to extend credentials.
    if (!(businessObjectDataEntity.getStatus().getCode().equals(BusinessObjectDataStatusEntity.UPLOADING))) {
        throw new IllegalArgumentException(String.format(String.format("Business object data {%s} has a status of \"%s\" and must be \"%s\" to extend " + "credentials.", businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey), businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING)));
    }
    // Get the S3 managed "loading dock" storage entity and make sure it exists.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    String s3BucketName = storageHelper.getStorageBucketName(storageEntity);
    // Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, businessObjectDataEntity);
    // Validate that the storage unit contains exactly one storage file.
    assertHasOneStorageFile(storageUnitEntity);
    // Get the storage file entity.
    StorageFileEntity storageFileEntity = IterableUtils.get(storageUnitEntity.getStorageFiles(), 0);
    // Get the storage file path.
    String storageFilePath = storageFileEntity.getPath();
    String awsRoleArn = getStorageUploadRoleArn(storageEntity);
    Integer awsRoleDurationSeconds = getStorageUploadSessionDuration(storageEntity);
    String awsKmsKeyId = storageHelper.getStorageKmsKeyId(storageEntity);
    // Get the temporary security credentials to access S3_MANAGED_STORAGE.
    Credentials assumedSessionCredentials = stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), String.valueOf(businessObjectDataEntity.getId()), awsRoleArn, awsRoleDurationSeconds, createUploaderPolicy(s3BucketName, storageFilePath, awsKmsKeyId));
    // Create the response.
    UploadSingleCredentialExtensionResponse response = new UploadSingleCredentialExtensionResponse();
    response.setAwsAccessKey(assumedSessionCredentials.getAccessKeyId());
    response.setAwsSecretKey(assumedSessionCredentials.getSecretAccessKey());
    response.setAwsSessionToken(assumedSessionCredentials.getSessionToken());
    response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(assumedSessionCredentials.getExpiration()));
    return response;
}

Example 29 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class UploadDownloadHelperServiceImpl method prepareForFileMoveImpl.

/**
 * Prepares to move an S3 file from the source bucket to the target bucket. On success, both the target and source business object data statuses are set to
 * "RE-ENCRYPTING" and the DTO is updated accordingly.
 *
 * @param objectKey the object key (i.e. filename)
 * @param completeUploadSingleParamsDto the DTO to be initialized with parameters required for complete upload single message processing
 */
protected void prepareForFileMoveImpl(String objectKey, CompleteUploadSingleParamsDto completeUploadSingleParamsDto) {
    try {
        // Obtain the source business object data entity.
        BusinessObjectDataEntity sourceBusinessObjectDataEntity = storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData();
        // Get the status and key of the source business object data entity.
        completeUploadSingleParamsDto.setSourceOldStatus(sourceBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setSourceBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity));
        // Find the target business object data by the source business object data's partition value, which should have been an UUID.
        // This is assuming that the target has the same partition value as the source, and that there exist one and only one target
        // business object data for this UUID.
        BusinessObjectDataEntity targetBusinessObjectDataEntity = getTargetBusinessObjectDataEntity(sourceBusinessObjectDataEntity);
        // Get the status and key of the target business object data entity.
        completeUploadSingleParamsDto.setTargetOldStatus(targetBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setTargetBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity));
        // This check effectively discards any duplicate SQS messages coming from S3 for the same uploaded file.
        for (BusinessObjectDataEntity businessObjectDataEntity : Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity)) {
            if (!BusinessObjectDataStatusEntity.UPLOADING.equals(businessObjectDataEntity.getStatus().getCode())) {
                LOGGER.info("Ignoring S3 notification since business object data status \"{}\" does not match the expected status \"{}\". " + "businessObjectDataKey={}", businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING, jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
                // method skip the rest of the steps required to complete the upload single message processing.
                return;
            }
        }
        // Get the S3 managed "loading dock" storage entity and make sure it exists.
        StorageEntity s3ManagedLoadingDockStorageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
        // Get bucket name for S3 managed "loading dock" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setSourceBucketName(storageHelper.getStorageBucketName(s3ManagedLoadingDockStorageEntity));
        // Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
        StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
        // Get the storage file entity.
        StorageFileEntity sourceStorageFileEntity = IterableUtils.get(sourceStorageUnitEntity.getStorageFiles(), 0);
        // Get the source storage file path.
        completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath());
        // Get the AWS parameters.
        AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
        completeUploadSingleParamsDto.setAwsParams(awsParamsDto);
        // Validate the source S3 file.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()).withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()).withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build();
        s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes());
        // Get the S3 managed "external" storage entity and make sure it exists.
        StorageEntity s3ManagedExternalStorageEntity = getUniqueStorage(targetBusinessObjectDataEntity);
        // Get bucket name for S3 managed "external" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setTargetBucketName(storageHelper.getStorageBucketName(s3ManagedExternalStorageEntity));
        // Get AWS KMS External Key ID.
        completeUploadSingleParamsDto.setKmsKeyId(storageHelper.getStorageKmsKeyId(s3ManagedExternalStorageEntity));
        // Make sure the target does not already contain the file.
        completeUploadSingleParamsDto.setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath());
        assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
        try {
            // Change the status of the source and target business object data to RE-ENCRYPTING.
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        }// caught by a business object data status check that occurs inside the prepareForFileMove() helper method.
         catch (OptimisticLockException e) {
            LOGGER.info("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey={} targetBusinessObjectDataKey={}", jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), jsonHelper.objectToJson(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
            // method skip the rest of the steps required to complete the upload single message processing.
            return;
        }
        // Set new status for the source and target business object data in the DTO.
        completeUploadSingleParamsDto.setSourceNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        completeUploadSingleParamsDto.setTargetNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    } catch (RuntimeException e) {
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setSourceNewStatus(setAndReturnNewSourceBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()));
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setTargetNewStatus(setAndReturnNewTargetBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
        // Delete the source S3 file. Please note that the method below only logs runtime exceptions without re-throwing them.
        deleteSourceS3ObjectAfterError(completeUploadSingleParamsDto.getSourceBucketName(), completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBusinessObjectDataKey());
        // Log the error.
        LOGGER.error("Failed to process upload single completion request for file. s3Key=\"{}\"", objectKey, e);
    }
    // If a status update occurred for the source business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getSourceNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getSourceBusinessObjectDataKey(), completeUploadSingleParamsDto.getSourceNewStatus(), completeUploadSingleParamsDto.getSourceOldStatus());
    }
    // If a status update occurred for the target business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getTargetNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getTargetBusinessObjectDataKey(), completeUploadSingleParamsDto.getTargetNewStatus(), completeUploadSingleParamsDto.getTargetOldStatus());
    }
}

Example 30 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceTest method testPrepareToFinalizeRestoreMissingOptionalParameters.

@Test
public void testPrepareToFinalizeRestoreMissingOptionalParameters() throws Exception {
    // Create a business object data key without sub-partition values.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create database entities required for testing.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataServiceTestHelper.createDatabaseEntitiesForFinalizeRestoreTesting(businessObjectDataKey);
    // Get the storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = storageUnitHelper.createStorageUnitKey(businessObjectDataKey, STORAGE_NAME);
    // Prepare to finalize a restore for business object data without sub-partition values.
    BusinessObjectDataRestoreDto result = businessObjectDataFinalizeRestoreHelperService.prepareToFinalizeRestore(storageUnitKey);
    // Validate the returned object.
    assertEquals(new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, NO_S3_ENDPOINT, S3_BUCKET_NAME, result.getS3KeyPrefix(), NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles()), NO_EXCEPTION), result);
}