Examples with StorageEntity org.finra.herd.model.jpa.StorageEntity used on opensource projects

Example 31 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageServiceImpl method deleteStorage.

@Override
public Storage deleteStorage(StorageKey storageKey) {
    // Perform validation and trim.
    validateAndTrimStorageKey(storageKey);
    // Retrieve and ensure that a storage exists.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageKey);
    // Delete the storage.
    storageDao.delete(storageEntity);
    // Return the storage information.
    return createStorageFromEntity(storageEntity);
}

Example 32 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageUnitServiceImpl method getS3KeyPrefixImpl.

/**
 * Gets the S3 key prefix.
 *
 * @param businessObjectDataKey the business object data key
 * @param businessObjectFormatPartitionKey the business object format partition key
 * @param storageName the storage name
 * @param createNewVersion specifies if it is OK to return an S3 key prefix for a new business object data version that is not an initial version. This
 * parameter is ignored, when the business object data version is specified.
 *
 * @return the S3 key prefix
 */
protected S3KeyPrefixInformation getS3KeyPrefixImpl(BusinessObjectDataKey businessObjectDataKey, String businessObjectFormatPartitionKey, String storageName, Boolean createNewVersion) {
    // Validate and trim the business object data key.
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, false);
    // If specified, trim the partition key parameter.
    String businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKey;
    if (businessObjectFormatPartitionKeyLocal != null) {
        businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKeyLocal.trim();
    }
    // If specified, trim the storage name. Otherwise, default to the configuration option.
    String storageNameLocal = storageName;
    if (StringUtils.isNotBlank(storageNameLocal)) {
        storageNameLocal = storageNameLocal.trim();
    } else {
        storageNameLocal = configurationHelper.getProperty(ConfigurationValue.S3_STORAGE_NAME_DEFAULT);
    }
    // Get the business object format for the specified parameters and make sure it exists.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoHelper.getBusinessObjectFormatEntity(new BusinessObjectFormatKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion()));
    // If specified, ensure that partition key matches what's configured within the business object format.
    if (StringUtils.isNotBlank(businessObjectFormatPartitionKeyLocal)) {
        Assert.isTrue(businessObjectFormatEntity.getPartitionKey().equalsIgnoreCase(businessObjectFormatPartitionKeyLocal), "Partition key \"" + businessObjectFormatPartitionKeyLocal + "\" doesn't match configured business object format partition key \"" + businessObjectFormatEntity.getPartitionKey() + "\".");
    }
    // Get and validate the storage along with the relative attributes.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageNameLocal);
    // If the business object data version is not specified, get the next business object data version value.
    if (businessObjectDataKey.getBusinessObjectDataVersion() == null) {
        // Get the latest data version for this business object data, if it exists.
        BusinessObjectDataEntity latestVersionBusinessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(new BusinessObjectDataKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion(), businessObjectDataKey.getPartitionValue(), businessObjectDataKey.getSubPartitionValues(), null));
        // Throw an error if this business object data already exists and createNewVersion flag is not set.
        if (latestVersionBusinessObjectDataEntity != null && !createNewVersion) {
            throw new AlreadyExistsException("Initial version of the business object data already exists.");
        }
        businessObjectDataKey.setBusinessObjectDataVersion(latestVersionBusinessObjectDataEntity == null ? BusinessObjectDataEntity.BUSINESS_OBJECT_DATA_INITIAL_VERSION : latestVersionBusinessObjectDataEntity.getVersion() + 1);
    }
    // Build the S3 key prefix string.
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey);
    // Create and return the S3 key prefix.
    S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation();
    s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix);
    return s3KeyPrefixInformation;
}

Example 33 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class UploadDownloadHelperServiceImpl method prepareForFileMoveImpl.

/**
 * Prepares to move an S3 file from the source bucket to the target bucket. On success, both the target and source business object data statuses are set to
 * "RE-ENCRYPTING" and the DTO is updated accordingly.
 *
 * @param objectKey the object key (i.e. filename)
 * @param completeUploadSingleParamsDto the DTO to be initialized with parameters required for complete upload single message processing
 */
protected void prepareForFileMoveImpl(String objectKey, CompleteUploadSingleParamsDto completeUploadSingleParamsDto) {
    try {
        // Obtain the source business object data entity.
        BusinessObjectDataEntity sourceBusinessObjectDataEntity = storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData();
        // Get the status and key of the source business object data entity.
        completeUploadSingleParamsDto.setSourceOldStatus(sourceBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setSourceBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity));
        // Find the target business object data by the source business object data's partition value, which should have been an UUID.
        // This is assuming that the target has the same partition value as the source, and that there exist one and only one target
        // business object data for this UUID.
        BusinessObjectDataEntity targetBusinessObjectDataEntity = getTargetBusinessObjectDataEntity(sourceBusinessObjectDataEntity);
        // Get the status and key of the target business object data entity.
        completeUploadSingleParamsDto.setTargetOldStatus(targetBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setTargetBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity));
        // This check effectively discards any duplicate SQS messages coming from S3 for the same uploaded file.
        for (BusinessObjectDataEntity businessObjectDataEntity : Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity)) {
            if (!BusinessObjectDataStatusEntity.UPLOADING.equals(businessObjectDataEntity.getStatus().getCode())) {
                LOGGER.info("Ignoring S3 notification since business object data status \"{}\" does not match the expected status \"{}\". " + "businessObjectDataKey={}", businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING, jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
                // method skip the rest of the steps required to complete the upload single message processing.
                return;
            }
        }
        // Get the S3 managed "loading dock" storage entity and make sure it exists.
        StorageEntity s3ManagedLoadingDockStorageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
        // Get bucket name for S3 managed "loading dock" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setSourceBucketName(storageHelper.getStorageBucketName(s3ManagedLoadingDockStorageEntity));
        // Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
        StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
        // Get the storage file entity.
        StorageFileEntity sourceStorageFileEntity = IterableUtils.get(sourceStorageUnitEntity.getStorageFiles(), 0);
        // Get the source storage file path.
        completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath());
        // Get the AWS parameters.
        AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
        completeUploadSingleParamsDto.setAwsParams(awsParamsDto);
        // Validate the source S3 file.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()).withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()).withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build();
        s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes());
        // Get the S3 managed "external" storage entity and make sure it exists.
        StorageEntity s3ManagedExternalStorageEntity = getUniqueStorage(targetBusinessObjectDataEntity);
        // Get bucket name for S3 managed "external" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setTargetBucketName(storageHelper.getStorageBucketName(s3ManagedExternalStorageEntity));
        // Get AWS KMS External Key ID.
        completeUploadSingleParamsDto.setKmsKeyId(storageHelper.getStorageKmsKeyId(s3ManagedExternalStorageEntity));
        // Make sure the target does not already contain the file.
        completeUploadSingleParamsDto.setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath());
        assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
        try {
            // Change the status of the source and target business object data to RE-ENCRYPTING.
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        }// caught by a business object data status check that occurs inside the prepareForFileMove() helper method.
         catch (OptimisticLockException e) {
            LOGGER.info("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey={} targetBusinessObjectDataKey={}", jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), jsonHelper.objectToJson(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
            // method skip the rest of the steps required to complete the upload single message processing.
            return;
        }
        // Set new status for the source and target business object data in the DTO.
        completeUploadSingleParamsDto.setSourceNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        completeUploadSingleParamsDto.setTargetNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    } catch (RuntimeException e) {
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setSourceNewStatus(setAndReturnNewSourceBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()));
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setTargetNewStatus(setAndReturnNewTargetBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
        // Delete the source S3 file. Please note that the method below only logs runtime exceptions without re-throwing them.
        deleteSourceS3ObjectAfterError(completeUploadSingleParamsDto.getSourceBucketName(), completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBusinessObjectDataKey());
        // Log the error.
        LOGGER.error("Failed to process upload single completion request for file. s3Key=\"{}\"", objectKey, e);
    }
    // If a status update occurred for the source business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getSourceNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getSourceBusinessObjectDataKey(), completeUploadSingleParamsDto.getSourceNewStatus(), completeUploadSingleParamsDto.getSourceOldStatus());
    }
    // If a status update occurred for the target business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getTargetNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getTargetBusinessObjectDataKey(), completeUploadSingleParamsDto.getTargetNewStatus(), completeUploadSingleParamsDto.getTargetOldStatus());
    }
}

Example 34 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class RelationalTableRegistrationHelperServiceImpl method getRelationalStorageAttributesImpl.

/**
 * Gets storage attributes required to perform relation table registration. This method also validates database entities per specified relational table
 * registration create request.
 *
 * @param relationalTableRegistrationCreateRequest the relational table registration create request
 * @param appendToExistingBusinessObjectDefinition boolean flag that determines if the format should be appended to an existing business object definition
 *
 * @return the relational storage attributes DtO
 */
RelationalStorageAttributesDto getRelationalStorageAttributesImpl(RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest, Boolean appendToExistingBusinessObjectDefinition) {
    // Validate that specified namespace exists.
    namespaceDaoHelper.getNamespaceEntity(relationalTableRegistrationCreateRequest.getNamespace());
    // Create a business object definition key.
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(relationalTableRegistrationCreateRequest.getNamespace(), relationalTableRegistrationCreateRequest.getBusinessObjectDefinitionName());
    // Ensure that a business object definition with the specified key doesn't already exist.
    if (BooleanUtils.isNotTrue(appendToExistingBusinessObjectDefinition) && businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(businessObjectDefinitionKey) != null) {
        throw new AlreadyExistsException(String.format("Business object definition with name \"%s\" already exists for namespace \"%s\".", businessObjectDefinitionKey.getBusinessObjectDefinitionName(), businessObjectDefinitionKey.getNamespace()));
    }
    // Get the latest format version for this business format, if it exists.
    BusinessObjectFormatEntity latestVersionBusinessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(relationalTableRegistrationCreateRequest.getNamespace(), relationalTableRegistrationCreateRequest.getBusinessObjectDefinitionName(), relationalTableRegistrationCreateRequest.getBusinessObjectFormatUsage(), FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE, null));
    // If the latest version exists, fail with an already exists exception.
    if (latestVersionBusinessObjectFormatEntity != null) {
        throw new AlreadyExistsException(String.format("Format with file type \"%s\" and usage \"%s\" already exists for business object definition \"%s\".", latestVersionBusinessObjectFormatEntity.getFileType().getCode(), latestVersionBusinessObjectFormatEntity.getUsage(), latestVersionBusinessObjectFormatEntity.getBusinessObjectDefinition().getName()));
    }
    // Validate that specified data provider exists.
    dataProviderDaoHelper.getDataProviderEntity(relationalTableRegistrationCreateRequest.getDataProviderName());
    // Get the storage.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(relationalTableRegistrationCreateRequest.getStorageName());
    // Only RELATIONAL storage platform is supported for the relational table registration feature.
    Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.RELATIONAL), String.format("Cannot register relational table in \"%s\" storage of %s storage platform type. Only %s storage platform type is supported by this feature.", storageEntity.getName(), storageEntity.getStoragePlatform().getName(), StoragePlatformEntity.RELATIONAL));
    // Get JDBC URL for this storage. This storage attribute is required and must have a non-blank value.
    String jdbcUrl = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.STORAGE_ATTRIBUTE_NAME_JDBC_URL), storageEntity, true, true);
    // Get JDBC username for this storage. This storage attribute is not required and it is allowed to have a blank value.
    String jdbcUsername = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.STORAGE_ATTRIBUTE_NAME_JDBC_USERNAME), storageEntity, false, false);
    // Get JDBC user credential name for this storage. This storage attribute is not required and it is allowed to have a blank value.
    String jdbcUserCredentialName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.STORAGE_ATTRIBUTE_NAME_JDBC_USER_CREDENTIAL_NAME), storageEntity, false, false);
    // Create and initialize a relational storage attributes DTO.
    RelationalStorageAttributesDto relationalStorageAttributesDto = new RelationalStorageAttributesDto();
    relationalStorageAttributesDto.setJdbcUrl(jdbcUrl);
    relationalStorageAttributesDto.setJdbcUsername(jdbcUsername);
    relationalStorageAttributesDto.setJdbcUserCredentialName(jdbcUserCredentialName);
    return relationalStorageAttributesDto;
}

Example 35 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageUnitServiceImpl method createBusinessObjectDataStorageUnitImpl.

/**
 * Creates new storage unit for a given business object data and storage.
 *
 * @param request the create business object data storage unit create request
 *
 * @return the create business object data storage unit create response
 */
protected BusinessObjectDataStorageUnitCreateResponse createBusinessObjectDataStorageUnitImpl(BusinessObjectDataStorageUnitCreateRequest request) {
    // Validate the request.
    validateBusinessObjectDataStorageUnitCreateRequest(request);
    // Retrieve and validate that business object data exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(storageUnitHelper.getBusinessObjectDataKey(request.getBusinessObjectDataStorageUnitKey()));
    // Retrieve and validate that storage exists.
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(request.getBusinessObjectDataStorageUnitKey().getStorageName());
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = businessObjectDataDaoHelper.createStorageUnitEntity(businessObjectDataEntity, storageEntity, request.getStorageDirectory(), request.getStorageFiles(), request.isDiscoverStorageFiles());
    // Persist the newly created storage unit entity.
    storageUnitDao.saveAndRefresh(storageUnitEntity);
    // Construct and return the response.
    return createBusinessObjectDataStorageUnitCreateResponse(storageUnitEntity);
}