Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 11 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImplTest method testExecuteStoragePolicyTransitionImpl.

@Test
public void testExecuteStoragePolicyTransitionImpl() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage file path.
    String storageFilePath = TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create a list of storage files to be passed as an input.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a storage policy transition parameters DTO.
    StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto = new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create a list of S3 object summaries selected without zero byte directory markers.
    List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = Arrays.asList(new S3ObjectSummary());
    // Create a list of all S3 files matching the S3 key prefix form the S3 bucket.
    List<S3ObjectSummary> actualS3Files = Arrays.asList(new S3ObjectSummary());
    // Create a list of storage files selected for S3 object tagging.
    List<StorageFile> storageFilesSelectedForTagging = Arrays.asList(new StorageFile());
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForTagging = Arrays.asList(new File(storageFilePath));
    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    updatedS3FileTransferRequestParamsDto.setFiles(filesSelectedForTagging);
    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME)).thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(actualS3FilesWithoutZeroByteDirectoryMarkers);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(actualS3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesSelectedForTagging);
    when(storageFileHelper.getFiles(storageFilesSelectedForTagging)).thenReturn(filesSelectedForTagging);
    // Call the method under test.
    storagePolicyProcessorHelperServiceImpl.executeStoragePolicyTransitionImpl(storagePolicyTransitionParamsDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, actualS3FilesWithoutZeroByteDirectoryMarkers, STORAGE_NAME, businessObjectDataKey);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, false);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
    verify(storageFileHelper).getFiles(storageFilesSelectedForTagging);
    verify(s3Service).tagObjects(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME), storagePolicyTransitionParamsDto);
}

Example 12 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoImplTest method testTagObjectsS3ClientCreationFails.

@Test
public void testTagObjectsS3ClientCreationFails() {
    // Create an S3 key.
    String s3Key = S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create an S3 file transfer request parameters DTO to access S3 objects without specifying S3 files.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setFiles(Arrays.asList(new File(s3Key)));
    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenThrow(new AmazonServiceException(ERROR_MESSAGE));
    // Try to call the method under test.
    try {
        s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag);
    } catch (IllegalStateException e) {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key in \"%s\" bucket. Reason: %s (Service: null; Status Code: 0; Error Code: null; Request ID: null)", s3Key, S3_BUCKET_NAME, ERROR_MESSAGE), e.getMessage());
    }
    // Verify the external calls.
    verify(retryPolicyFactory).getRetryPolicy();
    verifyNoMoreInteractionsHelper();
}

Example 13 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceImpl method validateStorageFiles.

/**
 * Validates a list of storage files to be added to the specified storage unit.
 *
 * @param storageFiles the list of storage files
 * @param storageUnitEntity the storage unit entity
 * @param validatePathPrefix the validate path prefix flag
 * @param validateFileExistence the validate file existence flag
 * @param validateFileSize the validate file size flag
 */
private void validateStorageFiles(List<StorageFile> storageFiles, StorageUnitEntity storageUnitEntity, boolean validatePathPrefix, boolean validateFileExistence, boolean validateFileSize) {
    // Retrieve all storage files already registered for this storage unit loaded in a map for easy access.
    Map<String, StorageFileEntity> storageFileEntities = storageFileHelper.getStorageFileEntitiesMap(storageUnitEntity.getStorageFiles());
    // Perform validation of storage files listed in the request per storage directory path and/or validation flags.
    String directoryPath = null;
    String directoryPathWithTrailingSlash = null;
    if (StringUtils.isNotBlank(storageUnitEntity.getDirectoryPath())) {
        // Use the storage directory path from the storage unit.
        directoryPath = storageUnitEntity.getDirectoryPath();
        // Add a trailing slash to the storage directory path if it doesn't already have it.
        directoryPathWithTrailingSlash = StringUtils.appendIfMissing(directoryPath, "/");
        // If a storage directory path exists, then validate that all files being added are contained within that directory.
        for (StorageFile storageFile : storageFiles) {
            Assert.isTrue(storageFile.getFilePath().startsWith(directoryPathWithTrailingSlash), String.format("Storage file path \"%s\" does not match the storage directory path \"%s\".", storageFile.getFilePath(), directoryPathWithTrailingSlash));
        }
    } else if (validatePathPrefix || validateFileExistence) {
        // Use the expected S3 key prefix value as the storage directory path.
        directoryPath = s3KeyPrefixHelper.buildS3KeyPrefix(storageUnitEntity.getStorage(), storageUnitEntity.getBusinessObjectData().getBusinessObjectFormat(), businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData()));
        // Add a trailing slash to the expected S3 key prefix if it doesn't already have it.
        directoryPathWithTrailingSlash = StringUtils.appendIfMissing(directoryPath, "/");
        // Validate that all files are contained within the expected S3 key prefix.
        for (StorageFile storageFile : storageFiles) {
            Assert.isTrue(storageFile.getFilePath().startsWith(directoryPathWithTrailingSlash), String.format("Specified storage file path \"%s\" does not match the expected S3 key prefix \"%s\".", storageFile.getFilePath(), directoryPathWithTrailingSlash));
        }
    }
    // Validate that files in the request does not already exist in the database.
    if (StringUtils.isNotBlank(directoryPath)) {
        // Get a list of request storage file paths.
        List<String> requestStorageFilePaths = storageFileHelper.getFilePathsFromStorageFiles(storageFiles);
        // Retrieve all already registered storage files from the storage that start with the directory path.
        List<String> registeredStorageFilePaths = storageFileDao.getStorageFilesByStorageAndFilePathPrefix(storageUnitEntity.getStorage().getName(), directoryPathWithTrailingSlash);
        // Check if request contains any of the already registered files.
        registeredStorageFilePaths.retainAll(requestStorageFilePaths);
        if (!CollectionUtils.isEmpty(registeredStorageFilePaths)) {
            // Retrieve the storage file entity for the first "already registered" storage file.
            // Since the discovered storage file path exists in the database, we should not get a null back.
            StorageFileEntity storageFileEntity = storageFileDao.getStorageFileByStorageNameAndFilePath(storageUnitEntity.getStorage().getName(), registeredStorageFilePaths.get(0));
            // Throw an exception reporting the information on the "already registered" storage file.
            throw new AlreadyExistsException(String.format("S3 file \"%s\" in \"%s\" storage is already registered by the business object data {%s}.", registeredStorageFilePaths.get(0), storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(storageFileEntity.getStorageUnit().getBusinessObjectData())));
        }
    } else {
        // Since directory path is not available, we need to validate each storage file specified in the request individually.
        for (StorageFile storageFile : storageFiles) {
            // Ensure that the file is not already registered in this storage by some other business object data.
            StorageFileEntity storageFileEntity = storageFileDao.getStorageFileByStorageNameAndFilePath(storageUnitEntity.getStorage().getName(), storageFile.getFilePath());
            if (storageFileEntity != null) {
                throw new AlreadyExistsException(String.format("S3 file \"%s\" in \"%s\" storage is already registered by the business object data {%s}.", storageFile.getFilePath(), storageUnitEntity.getStorage().getName(), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(storageFileEntity.getStorageUnit().getBusinessObjectData())));
            }
        }
    }
    // Validate file existence.
    if (validateFileExistence) {
        // Get S3 bucket access parameters and set the key prefix to the directory path with a trailing slash.
        // Please note that since we got here, the directory path can not be empty.
        S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
        params.setS3KeyPrefix(directoryPathWithTrailingSlash);
        // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
        Map<String, StorageFile> actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
        // For the already registered storage files, validate each storage file against S3 keys and metadata reported by S3.
        for (Map.Entry<String, StorageFileEntity> entry : storageFileEntities.entrySet()) {
            storageFileHelper.validateStorageFileEntity(entry.getValue(), params.getS3BucketName(), actualS3Keys, validateFileSize);
        }
        // Validate each storage file listed in the request.
        for (StorageFile storageFile : storageFiles) {
            storageFileHelper.validateStorageFile(storageFile, params.getS3BucketName(), actualS3Keys, validateFileSize);
        }
    }
}

Example 14 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class UploadDownloadServiceImpl method initiateDownloadSingle.

@NamespacePermission(fields = "#namespace", permissions = NamespacePermissionEnum.READ)
@Override
public DownloadSingleInitiationResponse initiateDownloadSingle(String namespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, Integer businessObjectFormatVersion, String partitionValue, Integer businessObjectDataVersion) {
    // Create the business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(namespace, businessObjectDefinitionName, businessObjectFormatUsage, businessObjectFormatFileType, businessObjectFormatVersion, partitionValue, null, businessObjectDataVersion);
    // Validate the parameters
    businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    // Retrieve the persisted business object data
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey);
    // Make sure the status of the business object data is VALID
    businessObjectDataHelper.assertBusinessObjectDataStatusEquals(BusinessObjectDataStatusEntity.VALID, businessObjectDataEntity);
    // Get the external storage registered against this data
    // Validate that the storage unit exists
    StorageUnitEntity storageUnitEntity = IterableUtils.get(businessObjectDataEntity.getStorageUnits(), 0);
    // Validate that the storage unit contains only 1 file
    assertHasOneStorageFile(storageUnitEntity);
    String s3BucketName = storageHelper.getStorageBucketName(storageUnitEntity.getStorage());
    String s3ObjectKey = IterableUtils.get(storageUnitEntity.getStorageFiles(), 0).getPath();
    // Get the temporary credentials
    Credentials downloaderCredentials = getExternalDownloaderCredentials(storageUnitEntity.getStorage(), String.valueOf(businessObjectDataEntity.getId()), s3ObjectKey);
    // Generate a pre-signed URL
    Date expiration = downloaderCredentials.getExpiration();
    S3FileTransferRequestParamsDto s3BucketAccessParams = storageHelper.getS3BucketAccessParams(storageUnitEntity.getStorage());
    String presignedUrl = s3Dao.generateGetObjectPresignedUrl(s3BucketName, s3ObjectKey, expiration, s3BucketAccessParams);
    // Construct and return the response
    DownloadSingleInitiationResponse response = new DownloadSingleInitiationResponse();
    response.setBusinessObjectData(businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity));
    response.setAwsAccessKey(downloaderCredentials.getAccessKeyId());
    response.setAwsSecretKey(downloaderCredentials.getSecretAccessKey());
    response.setAwsSessionToken(downloaderCredentials.getSessionToken());
    response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(expiration));
    response.setPreSignedUrl(presignedUrl);
    return response;
}

Example 15 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class UploadDownloadServiceImpl method initiateDownloadSingleSampleFile.

@Override
public DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationResponse initiateDownloadSingleSampleFile(DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationRequest request) {
    // Validate and trim the request parameters.
    validateDownloadBusinessObjectDefinitionSampleDataFileSingleInitiationRequest(request);
    // Get the business object definition sample data file key.
    BusinessObjectDefinitionSampleDataFileKey businessObjectDefinitionSampleDataFileKey = request.getBusinessObjectDefinitionSampleDataFileKey();
    // Get the business object definition key.
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(businessObjectDefinitionSampleDataFileKey.getNamespace(), businessObjectDefinitionSampleDataFileKey.getBusinessObjectDefinitionName());
    // Get the business object definition entity and ensure it exists.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity = businessObjectDefinitionDaoHelper.getBusinessObjectDefinitionEntity(businessObjectDefinitionKey);
    // Get the sample data file exists for the business object definition and ensure it exists.
    BusinessObjectDefinitionSampleDataFileEntity businessObjectDefinitionSampleDataFileEntity = getBusinessObjectDefinitionSampleDataFileEntity(businessObjectDefinitionEntity, businessObjectDefinitionSampleDataFileKey);
    // Retrieve the storage related information.
    StorageEntity storageEntity = businessObjectDefinitionSampleDataFileEntity.getStorage();
    String s3BucketName = storageHelper.getStorageBucketName(storageEntity);
    String s3ObjectKey = businessObjectDefinitionSampleDataFileKey.getDirectoryPath() + businessObjectDefinitionSampleDataFileKey.getFileName();
    String sessionID = UUID.randomUUID().toString();
    // Get the temporary credentials.
    Credentials downloaderCredentials = getDownloaderCredentialsNoKmsKey(storageEntity, sessionID, s3ObjectKey);
    // Generate a pre-signed URL.
    Date expiration = downloaderCredentials.getExpiration();
    S3FileTransferRequestParamsDto s3BucketAccessParams = storageHelper.getS3BucketAccessParams(storageEntity);
    String presignedUrl = s3Dao.generateGetObjectPresignedUrl(s3BucketName, s3ObjectKey, expiration, s3BucketAccessParams);
    // Create the download business object definition sample data file single initiation response.
    DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationResponse response = new DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationResponse();
    response.setBusinessObjectDefinitionSampleDataFileKey(new BusinessObjectDefinitionSampleDataFileKey(businessObjectDefinitionEntity.getNamespace().getCode(), businessObjectDefinitionEntity.getName(), businessObjectDefinitionSampleDataFileEntity.getDirectoryPath(), businessObjectDefinitionSampleDataFileEntity.getFileName()));
    response.setAwsS3BucketName(s3BucketName);
    response.setAwsAccessKey(downloaderCredentials.getAccessKeyId());
    response.setAwsSecretKey(downloaderCredentials.getSecretAccessKey());
    response.setAwsSessionToken(downloaderCredentials.getSessionToken());
    response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(expiration));
    response.setPreSignedUrl(presignedUrl);
    // Return the response.
    return response;
}