Examples with UploaderInputManifestDto org.finra.herd.model.dto.UploaderInputManifestDto used on opensource projects

Example 11 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class UploaderController method performUpload.

/**
 * Executes the uploader workflow.
 *
 * @param regServerAccessParamsDto the DTO for the parameters required to communicate with the registration server
 * @param manifestPath the local path to the manifest file
 * @param params the S3 file transfer request parameters being used to pass the following arguments: <ul> <li><code>s3AccessKey</code> the S3 access key
 * <li><code>s3SecretKey</code> the S3 secret key <li><code>localPath</code> the local path to directory containing data files
 * <li><code>httpProxyHost</code> the HTTP proxy host <li><code>httpProxyPort</code> the HTTP proxy port <li><code>maxThreads</code> the maximum number of
 * threads to use for file transfer to S3< <li><code>useRrs</code> specifies whether S3 reduced redundancy storage option will be used when copying to S3
 * </ul>
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created
 * @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
 * @param maxRetryAttempts the maximum number of the business object data registration retry attempts
 * @param retryDelaySecs the delay in seconds between the business object data registration retry attempts
 *
 * @throws InterruptedException if the upload thread was interrupted.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws IOException if an I/O error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.")
public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params, Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs) throws InterruptedException, JAXBException, IOException, URISyntaxException {
    boolean cleanUpS3KeyPrefixOnFailure = false;
    BusinessObjectDataKey businessObjectDataKey = null;
    try {
        // Process manifest file
        UploaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
        String storageName = getStorageNameFromManifest(manifest);
        manifest.setStorageName(storageName);
        // Validate local files and prepare a list of source files to copy to S3.
        List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles());
        // Validate that we do not have duplicate files listed in the manifest file.
        List<File> duplicateFiles = findDuplicateFiles(sourceFiles);
        if (!duplicateFiles.isEmpty()) {
            throw new IllegalArgumentException(String.format("Manifest contains duplicate file names. Duplicates: [\"%s\"]", StringUtils.join(duplicateFiles, "\", \"")));
        }
        // Initialize uploader web client.
        uploaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
        // Handle the latest business object data version if one exists.
        checkLatestBusinessObjectDataVersion(manifest, force);
        // Pre-register a new version of business object data in UPLOADING state with the registration server.
        BusinessObjectData businessObjectData = uploaderWebClient.preRegisterBusinessObjectData(manifest, storageName, createNewVersion);
        // Get business object data key.
        businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
        // Get the business object data version.
        Integer businessObjectDataVersion = businessObjectDataKey.getBusinessObjectDataVersion();
        // Add credential provider.
        params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() {

            @Override
            public AwsCredential getNewAwsCredential() throws Exception {
                return uploaderWebClient.getBusinessObjectDataUploadCredential(manifest, storageName, businessObjectDataVersion, null).getAwsCredential();
            }
        });
        // Get S3 key prefix from the business object data pre-registration response.
        String s3KeyPrefix = IterableUtils.get(businessObjectData.getStorageUnits(), 0).getStorageDirectory().getDirectoryPath();
        // Get S3 bucket information.
        Storage storage = uploaderWebClient.getStorage(storageName);
        // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
        String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
        // Set the KMS ID, if available
        String kmsKeyId = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storage, false);
        params.setKmsKeyId(kmsKeyId);
        // Special handling for the maxThreads command line option.
        params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS));
        // Populate several missing fields in the S3 file transfer request parameters DTO.
        params.setS3BucketName(s3BucketName);
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        params.setS3KeyPrefix(s3KeyPrefix + "/");
        params.setFiles(sourceFiles);
        // When listing S3 files, by default, we do not ignore 0 byte objects that represent S3 directories.
        if (s3Service.listDirectory(params).isEmpty()) {
            cleanUpS3KeyPrefixOnFailure = true;
        } else {
            throw new IllegalStateException(String.format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(), params.getS3KeyPrefix()));
        }
        // Upload files.
        s3Service.uploadFileList(params);
        // Get the list of files uploaded to S3 key prefix.
        if (LOGGER.isInfoEnabled()) {
            logS3KeyPrefixContents(params);
        }
        // Add storage files to the business object data.
        addStorageFilesWithRetry(businessObjectDataKey, manifest, params, storage.getName(), maxRetryAttempts, retryDelaySecs);
        // Change status of the business object data to VALID.
        uploaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) {
        // occurred, let's rollback the data transfer (clean up the S3 key prefix).
        if (cleanUpS3KeyPrefixOnFailure) {
            LOGGER.info(String.format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(), params.getS3BucketName()));
            s3Service.deleteDirectoryIgnoreException(params);
        }
        // If a new business object data version got pre-registered, update it's status to INVALID.
        if (businessObjectDataKey != null) {
            uploaderWebClient.updateBusinessObjectDataStatusIgnoreException(businessObjectDataKey, BusinessObjectDataStatusEntity.INVALID);
        }
        throw e;
    }
}

Example 12 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadManifestContainsDuplicateFileNames.

@Test(expected = IllegalArgumentException.class)
public void testPerformUploadManifestContainsDuplicateFileNames() throws Exception {
    List<ManifestFile> duplicateTestDataFiles = new ArrayList<>();
    duplicateTestDataFiles.addAll(testManifestFiles);
    duplicateTestDataFiles.add(testManifestFiles.get(0));
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : testManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(duplicateTestDataFiles);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Try to upload business object data having duplicate file names.
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}

Example 13 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadManifestFileNameDoesNotMatchActualFileName.

@Test(expected = IllegalArgumentException.class)
public void testPerformUploadManifestFileNameDoesNotMatchActualFileName() throws Exception {
    List<ManifestFile> declaredManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "test-data-2.txt"), FILE_SIZE_1_KB);
    List<ManifestFile> actualManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "TEST-DATA-2.TXT"), FILE_SIZE_1_KB);
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : actualManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory.
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(declaredManifestFiles);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Try to upload business object data with one of the file having name that does not match to incorrect name specified.
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}

Example 14 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class AbstractDataBridgeTest method getTestUploaderInputManifestDto.

/**
 * Returns an instance of the uploader input manifest object initialized per hard coded test values.
 *
 * @return the resulting UploaderManifest instance
 */
protected UploaderInputManifestDto getTestUploaderInputManifestDto(String partitionValue, List<String> subPartitionValues, boolean includeParents) {
    UploaderInputManifestDto manifest = new UploaderInputManifestDto();
    manifest.setNamespace(TEST_NAMESPACE);
    manifest.setBusinessObjectDefinitionName(TEST_BUSINESS_OBJECT_DEFINITION);
    manifest.setBusinessObjectFormatUsage(TEST_BUSINESS_OBJECT_FORMAT_USAGE);
    manifest.setBusinessObjectFormatFileType(TEST_BUSINESS_OBJECT_FORMAT_FILE_TYPE);
    manifest.setBusinessObjectFormatVersion(TEST_BUSINESS_OBJECT_FORMAT_VERSION.toString());
    manifest.setPartitionKey(TEST_BUSINESS_OBJECT_FORMAT_PARTITION_KEY);
    manifest.setPartitionValue(partitionValue);
    manifest.setSubPartitionValues(subPartitionValues);
    manifest.setManifestFiles(testManifestFiles);
    // Add attributes to the uploader manifest.
    HashMap<String, String> attributes = new HashMap<>();
    manifest.setAttributes(attributes);
    for (Attribute attribute : getTestAttributes()) {
        attributes.put(attribute.getName(), attribute.getValue());
    }
    // Add business object data parents.
    if (includeParents) {
        manifest.setBusinessObjectDataParents(getTestBusinessObjectDataParents());
    }
    return manifest;
}

Example 15 with UploaderInputManifestDto

use of org.finra.herd.model.dto.UploaderInputManifestDto in project herd by FINRAOS.

the class AbstractDataBridgeTest method uploadAndRegisterTestDataParent.

/**
 * Uploads and registers a version if of the test business object data that will be used as a parent.
 *
 * @param s3KeyPrefix the destination S3 key prefix that must comply with the S3 naming conventions including the expected data version value
 * @param dataBridgeWebClient the databridge web client instance
 */
protected void uploadAndRegisterTestDataParent(String s3KeyPrefix, DataBridgeWebClient dataBridgeWebClient) throws Exception {
    uploadTestDataFilesToS3(s3KeyPrefix, testManifestFiles, new ArrayList<String>());
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto(TEST_PARENT_PARTITION_VALUE, TEST_SUB_PARTITION_VALUES, false);
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/");
    BusinessObjectData businessObjectData = dataBridgeWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, true);
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
    dataBridgeWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, s3FileTransferRequestParamsDto, StorageEntity.MANAGED_STORAGE);
    dataBridgeWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    // Clean up the local input directory used for the test data files upload.
    FileUtils.cleanDirectory(LOCAL_TEMP_PATH_INPUT.toFile());
}