Examples with UploaderInputManifestDto org.finra.herd.model.dto.UploaderInputManifestDto used on opensource projects

Example 1 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class DataBridgeWebClientTest method testAddStorageFilesResponse200BadContentReturnsNull.

@Test
public void testAddStorageFilesResponse200BadContentReturnsNull() throws Exception {
    dataBridgeWebClient.regServerAccessParamsDto.setRegServerHost(MockHttpClientOperationsImpl.HOSTNAME_RESPOND_WITH_STATUS_CODE_200_AND_INVALID_CONTENT);
    dataBridgeWebClient.regServerAccessParamsDto.setUseSsl(false);
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey();
    UploaderInputManifestDto manifest = getUploaderInputManifestDto();
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    String storageName = "testStorage";
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse = dataBridgeWebClient.addStorageFiles(businessObjectDataKey, manifest, s3FileTransferRequestParamsDto, storageName);
    assertNull("businessObjectDataStorageFilesCreateResponse", businessObjectDataStorageFilesCreateResponse);
}

Example 2 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class DownloaderControllerTest method testPerformDownloadBusinessObjectDataHasAttributes.

@Test
public void testPerformDownloadBusinessObjectDataHasAttributes() throws Exception {
    // Add attributes to the test business object data.
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    HashMap<String, String> attributes = new HashMap<>();
    uploaderInputManifestDto.setAttributes(attributes);
    attributes.put(ATTRIBUTE_NAME_1_MIXED_CASE, ATTRIBUTE_VALUE_1);
    attributes.put(ATTRIBUTE_NAME_2_MIXED_CASE, ATTRIBUTE_VALUE_2);
    // Prepare test data and runs a normal download scenario.
    runDownload();
}

Example 3 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class DownloaderWebClientTest method testGetS3KeyPrefix.

@Test
public void testGetS3KeyPrefix() throws Exception {
    // Upload and register business object data parents.
    uploadAndRegisterTestDataParents(downloaderWebClient);
    // Upload and register the initial version if of the test business object data.
    uploadTestDataFilesToS3(S3_TEST_PATH_V0);
    final UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    executeWithoutLogging(DataBridgeWebClient.class, new Command() {

        @Override
        public void execute() throws Exception {
            BusinessObjectData businessObjectData = downloaderWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, false);
            BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
            downloaderWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, getTestS3FileTransferRequestParamsDto(S3_TEST_PATH_V0 + "/"), StorageEntity.MANAGED_STORAGE);
            downloaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
        }
    });
    // Get S3 key prefix.
    BusinessObjectData businessObjectData = toBusinessObjectData(uploaderInputManifestDto);
    S3KeyPrefixInformation resultS3KeyPrefixInformation = downloaderWebClient.getS3KeyPrefix(businessObjectData);
    // Validate the results.
    assertNotNull(resultS3KeyPrefixInformation);
    assertEquals(S3_SIMPLE_TEST_PATH, resultS3KeyPrefixInformation.getS3KeyPrefix());
}

Example 4 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class AbstractDownloaderTest method uploadAndRegisterTestData.

/**
 * Uploads and registers a version if of the test business object data with the specified data files.
 *
 * @param s3KeyPrefix the destination S3 key prefix that must comply with the S3 naming conventions including the expected data version value
 * @param manifestFiles the test data files to be uploaded to S3 and registered
 * @param directoryPaths the list of directory paths to be created in S3 relative to the S3 key prefix
 */
protected void uploadAndRegisterTestData(String s3KeyPrefix, List<ManifestFile> manifestFiles, List<String> directoryPaths) throws Exception {
    uploadTestDataFilesToS3(s3KeyPrefix, manifestFiles, directoryPaths);
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(manifestFiles);
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/");
    BusinessObjectData businessObjectData = downloaderWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, false);
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
    downloaderWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, s3FileTransferRequestParamsDto, StorageEntity.MANAGED_STORAGE);
    downloaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    // Clean up the local input directory used for the test data files upload.
    FileUtils.cleanDirectory(LOCAL_TEMP_PATH_INPUT.toFile());
}

Example 5 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class UploaderControllerTest method runUpload.

/**
 * Runs a normal upload scenario.
 *
 * @param numOfThreads the maximum number of threads to use for file transfer to S3
 * @param attributes the attributes to be associated with the test data being uploaded
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created
 * @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
 * @param hostname optional override of the default web service hostname.
 * @param storageName optional storage name
 */
protected void runUpload(Integer numOfThreads, HashMap<String, String> attributes, Boolean createNewVersion, Boolean force, String hostname, String storageName) throws Exception {
    String hostnameToUse = hostname == null ? WEB_SERVICE_HOSTNAME : hostname;
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : testManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setAttributes(attributes);
    uploaderInputManifestDto.setStorageName(storageName);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Perform the upload.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_INPUT.toString());
    s3FileTransferRequestParamsDto.setMaxThreads(numOfThreads);
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(hostnameToUse).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, createNewVersion, force, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}