Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 6 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class UploaderApp method go.

/**
 * Parses the command line arguments and calls the controller to process the upload.
 *
 * @param args the command line arguments passed to the program.
 *
 * @return the return value of the application.
 * @throws Exception if there are problems performing the upload.
 */
@Override
public ReturnValue go(String[] args) throws Exception {
    // Create the Spring application context.
    ApplicationContext applicationContext = createApplicationContext();
    // Parse the command line arguments and return a return value if we shouldn't continue processing (e.g. we displayed usage information, etc.).
    ReturnValue returnValue = parseCommandLineArguments(args, applicationContext);
    if (returnValue != null) {
        return returnValue;
    }
    // Create an instance of S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto params = S3FileTransferRequestParamsDto.builder().withLocalPath(argParser.getStringValue(localPathOpt)).withUseRrs(argParser.getBooleanValue(rrsOpt)).withAwsAccessKeyId(argParser.getStringValue(s3AccessKeyOpt)).withAwsSecretKey(argParser.getStringValue(s3SecretKeyOpt)).withS3Endpoint(argParser.getStringValue(s3EndpointOpt)).withMaxThreads(maxThreads).withHttpProxyHost(argParser.getStringValue(httpProxyHostOpt)).withHttpProxyPort(httpProxyPort).withSocketTimeout(argParser.getIntegerValue(socketTimeoutOpt)).build();
    // Call the controller with the user specified parameters to perform the upload.
    UploaderController controller = applicationContext.getBean(UploaderController.class);
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(regServerHost).withRegServerPort(regServerPort).withUseSsl(useSsl).withUsername(argParser.getStringValue(usernameOpt)).withPassword(argParser.getStringValue(passwordOpt)).build();
    controller.performUpload(regServerAccessParamsDto, argParser.getFileValue(manifestPathOpt), params, argParser.getBooleanValue(createNewVersionOpt), argParser.getBooleanValue(forceOpt), maxRetryAttempts, retryDelaySecs);
    // No exceptions were returned so return success.
    return ReturnValue.SUCCESS;
}

Example 7 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class DownloaderControllerTest method runDownload.

/**
 * Prepares test data and runs a normal download scenario.
 *
 * @param downloaderInputManifestDto the downloader input manifest object instance
 * @param localPath the local target directory
 * @param numOfThreads the maximum number of threads to use for file transfer to S3
 * @param hostname optional override of the default web service hostname.
 */
protected void runDownload(DownloaderInputManifestDto downloaderInputManifestDto, String localPath, Integer numOfThreads, String hostname) throws Exception {
    String hostnameToUse = hostname == null ? WEB_SERVICE_HOSTNAME : hostname;
    // Upload and register business object data parents.
    uploadAndRegisterTestDataParents(downloaderWebClient);
    // Upload and register the initial version if of the test business object data.
    uploadAndRegisterTestData(S3_TEST_PATH_V0);
    // Create a downloader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    File downloaderInputManifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), downloaderInputManifestDto);
    // Perform the download.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_TEST_PATH_V0);
    s3FileTransferRequestParamsDto.setLocalPath(localPath);
    s3FileTransferRequestParamsDto.setMaxThreads(numOfThreads);
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(hostnameToUse).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    downloaderController.performDownload(regServerAccessParamsDto, downloaderInputManifestFile, s3FileTransferRequestParamsDto);
}

Example 8 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class DownloaderControllerTest method testPerformDownloadZeroByteDirectoryMarkersPresent.

@Test
public void testPerformDownloadZeroByteDirectoryMarkersPresent() throws Exception {
    // Upload and register business object data parents.
    uploadAndRegisterTestDataParents(downloaderWebClient);
    // Upload and register the initial version if of the test business object data.
    uploadAndRegisterTestData(S3_TEST_PATH_V0, testManifestFiles, S3_DIRECTORY_MARKERS);
    // Create a downloader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    File downloaderInputManifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), getTestDownloaderInputManifestDto());
    // Adjust the S3 file transfer parameters to be passed to the downloader controller.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_OUTPUT.toString());
    s3FileTransferRequestParamsDto.setMaxThreads(DownloaderController.MIN_THREADS);
    // Perform the download.
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    downloaderController.performDownload(regServerAccessParamsDto, downloaderInputManifestFile, s3FileTransferRequestParamsDto);
}

Example 9 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class AbstractDownloaderTest method uploadAndRegisterTestData.

/**
 * Uploads and registers a version if of the test business object data with the specified data files.
 *
 * @param s3KeyPrefix the destination S3 key prefix that must comply with the S3 naming conventions including the expected data version value
 * @param manifestFiles the test data files to be uploaded to S3 and registered
 * @param directoryPaths the list of directory paths to be created in S3 relative to the S3 key prefix
 */
protected void uploadAndRegisterTestData(String s3KeyPrefix, List<ManifestFile> manifestFiles, List<String> directoryPaths) throws Exception {
    uploadTestDataFilesToS3(s3KeyPrefix, manifestFiles, directoryPaths);
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(manifestFiles);
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/");
    BusinessObjectData businessObjectData = downloaderWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, false);
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
    downloaderWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, s3FileTransferRequestParamsDto, StorageEntity.MANAGED_STORAGE);
    downloaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    // Clean up the local input directory used for the test data files upload.
    FileUtils.cleanDirectory(LOCAL_TEMP_PATH_INPUT.toFile());
}

Example 10 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class ExpireRestoredBusinessObjectDataHelperServiceImplTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT), new StorageFile(S3_KEY_2, FILE_SIZE_2, ROW_COUNT_2));
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, STORAGE_UNIT_STATUS_2, STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    // Create an initial instance of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an updated version of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    // Create a mock S3 object summary for S3 object that belongs to Glacier storage class.
    S3ObjectSummary glacierS3ObjectSummary = mock(S3ObjectSummary.class);
    when(glacierS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary standardS3ObjectSummary = mock(S3ObjectSummary.class);
    when(standardS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
    // Create a list of S3 files.
    List<S3ObjectSummary> s3Files = Arrays.asList(glacierS3ObjectSummary, standardS3ObjectSummary);
    // Create a list of S3 objects that belong to Glacier storage class.
    List<S3ObjectSummary> glacierS3Files = Arrays.asList(glacierS3ObjectSummary);
    // Create a list of storage files that represent S3 objects of Glacier storage class.
    List<StorageFile> glacierStorageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a list of files.
    List<File> files = Arrays.asList(new File(S3_KEY));
    // Create a final version of DTO for business object data restore parameters.
    S3FileTransferRequestParamsDto finalS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    finalS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    finalS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    finalS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    finalS3FileTransferRequestParamsDto.setFiles(files);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles);
    when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(files);
    // Call the method under test.
    expireRestoredBusinessObjectDataHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true));
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
    verify(storageFileHelper).getFiles(glacierStorageFiles);
    verify(s3Service).restoreObjects(finalS3FileTransferRequestParamsDto, 1);
    verifyNoMoreInteractionsHelper();
}