Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 26 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class UploaderControllerTest method runUpload.

/**
 * Runs a normal upload scenario.
 *
 * @param numOfThreads the maximum number of threads to use for file transfer to S3
 * @param attributes the attributes to be associated with the test data being uploaded
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created
 * @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
 * @param hostname optional override of the default web service hostname.
 * @param storageName optional storage name
 */
protected void runUpload(Integer numOfThreads, HashMap<String, String> attributes, Boolean createNewVersion, Boolean force, String hostname, String storageName) throws Exception {
    String hostnameToUse = hostname == null ? WEB_SERVICE_HOSTNAME : hostname;
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : testManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setAttributes(attributes);
    uploaderInputManifestDto.setStorageName(storageName);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Perform the upload.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_INPUT.toString());
    s3FileTransferRequestParamsDto.setMaxThreads(numOfThreads);
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(hostnameToUse).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, createNewVersion, force, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}

Example 27 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadRegistrationError.

/**
 * TODO: We need the herd web service mocking done and this test case rewritten, so it would fail right at the end of performUpload() method (on the
 * business object data registration step) and triggered the rollbackUpload() to occur.
 */
@Test(expected = RuntimeException.class)
public void testPerformUploadRegistrationError() throws Exception {
    // Upload and register business object data parents.
    uploadAndRegisterTestDataParents(uploaderWebClient);
    runUpload(UploaderController.MIN_THREADS);
    // Clean up the local directory.
    FileUtils.deleteDirectory(LOCAL_TEMP_PATH_INPUT.toFile());
    // Clean up the destination S3 folder.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(S3_TEST_PATH_V0);
    if (!s3Service.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) {
        s3Service.deleteDirectory(s3FileTransferRequestParamsDto);
    }
    runUpload(UploaderController.MIN_THREADS);
}

Example 28 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testGenerateGetObjectPresignedUrl.

/**
 * Asserts that calling generateGetObjectPresignedUrl() will return the expected mocked pre-signed URL.
 */
@Test
public void testGenerateGetObjectPresignedUrl() {
    String bucketName = "test_bucketName";
    String key = "test_key";
    Date expiration = new Date(12345L);
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    String result = s3Dao.generateGetObjectPresignedUrl(bucketName, key, expiration, s3FileTransferRequestParamsDto);
    Assert.assertEquals("result", "https://" + bucketName + "/" + key + "?method=GET&expiration=" + expiration.getTime(), result);
}

Example 29 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testGetObjectMetadataServiceException.

/**
 * Test S3 exception handling in the getObjectMetadata S3Dao operation.
 */
@Test
public void testGetObjectMetadataServiceException() {
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    // Validate that S3 service exception is handled correctly when retrieving S3 object metadata.
    try {
        s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY);
        s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto);
        fail("Should throw an IllegalStateException when Amazon service exception occurs.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Failed to get S3 metadata for object key \"%s\" from bucket \"%s\". " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", s3FileTransferRequestParamsDto.getS3KeyPrefix(), s3FileTransferRequestParamsDto.getS3BucketName()), e.getMessage());
    }
}

Example 30 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadFile.

/**
 * Test that we are able to perform the uploadFile S3Dao operation on S3 using our DAO tier.
 */
@Test
public void testUploadFile() throws IOException, InterruptedException {
    // Create local test file.
    File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_1_KB);
    Assert.assertTrue(targetFile.isFile());
    Assert.assertTrue(targetFile.length() == FILE_SIZE_1_KB);
    // Upload test file to s3Dao.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY);
    s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath());
    S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == 1L);
    // Validate the file upload.
    s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY));
}