Examples with DownloaderOutputManifestDto org.finra.herd.model.dto.DownloaderOutputManifestDto used on opensource projects

Example 1 with DownloaderOutputManifestDto

use of org.finra.herd.model.dto.DownloaderOutputManifestDto in project herd by FINRAOS.

the class DownloaderControllerTest method testCreateDownloaderOutputManifestDtoAssertOutputFilesEmptyWhenStorageFilesNull.

@Test
public void testCreateDownloaderOutputManifestDtoAssertOutputFilesEmptyWhenStorageFilesNull() {
    BusinessObjectData businessObjectData = new BusinessObjectData();
    StorageUnit storageUnit = new StorageUnit(new Storage("storageName", null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
    String s3KeyPrefix = "s3KeyPrefix";
    DownloaderOutputManifestDto actual = downloaderController.createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefix);
    assertEquals(0, actual.getManifestFiles().size());
}

Example 2 with DownloaderOutputManifestDto

use of org.finra.herd.model.dto.DownloaderOutputManifestDto in project herd by FINRAOS.

the class DownloaderController method createDownloaderOutputManifestDto.

/**
 * Creates a downloader output manifest instance from the business object data.
 *
 * @param businessObjectData the business object data that we need to create the manifest for
 * @param storageUnit the S3 storage unit for this business object data
 * @param s3KeyPrefix the S3 key prefix for this business object data
 *
 * @return the created downloader output manifest instance
 */
protected DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix) {
    DownloaderOutputManifestDto downloaderOutputManifestDto = new DownloaderOutputManifestDto();
    // Populate basic fields.
    downloaderOutputManifestDto.setNamespace(businessObjectData.getNamespace());
    downloaderOutputManifestDto.setBusinessObjectDefinitionName(businessObjectData.getBusinessObjectDefinitionName());
    downloaderOutputManifestDto.setBusinessObjectFormatUsage(businessObjectData.getBusinessObjectFormatUsage());
    downloaderOutputManifestDto.setBusinessObjectFormatFileType(businessObjectData.getBusinessObjectFormatFileType());
    downloaderOutputManifestDto.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    downloaderOutputManifestDto.setPartitionKey(businessObjectData.getPartitionKey());
    downloaderOutputManifestDto.setPartitionValue(businessObjectData.getPartitionValue());
    downloaderOutputManifestDto.setSubPartitionValues(businessObjectData.getSubPartitionValues());
    downloaderOutputManifestDto.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
    downloaderOutputManifestDto.setStorageName(storageUnit.getStorage().getName());
    // Build a list of manifest files with paths relative to the S3 key prefix.
    List<ManifestFile> manifestFiles = new ArrayList<>();
    downloaderOutputManifestDto.setManifestFiles(manifestFiles);
    if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) {
        for (StorageFile storageFile : storageUnit.getStorageFiles()) {
            ManifestFile manifestFile = new ManifestFile();
            manifestFiles.add(manifestFile);
            manifestFile.setFileName(storageFile.getFilePath().replace(s3KeyPrefix, ""));
            manifestFile.setFileSizeBytes(storageFile.getFileSizeBytes());
            manifestFile.setRowCount(storageFile.getRowCount());
        }
    }
    // Populate the attributes.
    HashMap<String, String> attributes = new HashMap<>();
    if (!CollectionUtils.isEmpty(businessObjectData.getAttributes())) {
        for (Attribute attribute : businessObjectData.getAttributes()) {
            attributes.put(attribute.getName(), attribute.getValue());
        }
    }
    downloaderOutputManifestDto.setAttributes(attributes);
    // Populate the business object data parents and children.
    downloaderOutputManifestDto.setBusinessObjectDataParents(businessObjectData.getBusinessObjectDataParents());
    downloaderOutputManifestDto.setBusinessObjectDataChildren(businessObjectData.getBusinessObjectDataChildren());
    return downloaderOutputManifestDto;
}

Example 3 with DownloaderOutputManifestDto

use of org.finra.herd.model.dto.DownloaderOutputManifestDto in project herd by FINRAOS.

the class DownloaderControllerTest method testCreateDownloaderOutputManifestDtoAssertOutputAttributesSetWhenBdataAttributesSet.

@Test
public void testCreateDownloaderOutputManifestDtoAssertOutputAttributesSetWhenBdataAttributesSet() {
    BusinessObjectData businessObjectData = new BusinessObjectData();
    businessObjectData.setAttributes(new ArrayList<>());
    businessObjectData.getAttributes().add(new Attribute("name", "value"));
    StorageUnit storageUnit = new StorageUnit(new Storage("storageName", null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
    String s3KeyPrefix = "s3KeyPrefix";
    DownloaderOutputManifestDto actual = downloaderController.createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefix);
    assertEquals(1, actual.getAttributes().size());
    assertEquals("value", actual.getAttributes().get("name"));
}

Example 4 with DownloaderOutputManifestDto

use of org.finra.herd.model.dto.DownloaderOutputManifestDto in project herd by FINRAOS.

the class DownloaderControllerTest method testCreateDownloaderOutputManifestDto.

@Test
public void testCreateDownloaderOutputManifestDto() {
    // Initiate input parameters.
    List<String> subPartitionValues = Arrays.asList("subPartitionValue1", "subPartitionValue2", "subPartitionValue3", "subPartitionValue4");
    String s3KeyPrefix = "s3KeyPrefix";
    StorageUnit storageUnit = new StorageUnit(new Storage("storageName", s3KeyPrefix, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
    Attribute attribute = new Attribute("name", "value");
    BusinessObjectData businessObjectData = new BusinessObjectData(1234, "businessObjectDefinitionNamespace", "businessObjectDefinitionName", "formatUsage", "formatFileType", 2345, "partitionKey", "partitionValue", subPartitionValues, 3456, true, BusinessObjectDataStatusEntity.VALID, Arrays.asList(storageUnit), Arrays.asList(attribute), new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
    // Create a downloader output manifest DTO.
    DownloaderOutputManifestDto resultDto = downloaderController.createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefix);
    // Validate the result DTO.
    assertEquals("businessObjectDefinitionNamespace", resultDto.getNamespace());
    assertEquals("businessObjectDefinitionName", resultDto.getBusinessObjectDefinitionName());
    assertEquals("formatUsage", resultDto.getBusinessObjectFormatUsage());
    assertEquals("formatFileType", resultDto.getBusinessObjectFormatFileType());
    assertEquals("2345", resultDto.getBusinessObjectFormatVersion());
    assertEquals("partitionKey", resultDto.getPartitionKey());
    assertEquals("partitionValue", resultDto.getPartitionValue());
    assertEquals(subPartitionValues, resultDto.getSubPartitionValues());
    assertEquals("3456", resultDto.getBusinessObjectDataVersion());
    assertEquals("storageName", resultDto.getStorageName());
    HashMap<String, String> expectedAttributes = new HashMap<>();
    expectedAttributes.put(attribute.getName(), attribute.getValue());
    assertEquals(expectedAttributes, resultDto.getAttributes());
    assertEquals(new ArrayList<>(), resultDto.getBusinessObjectDataParents());
    assertEquals(new ArrayList<>(), resultDto.getBusinessObjectDataChildren());
}

Example 5 with DownloaderOutputManifestDto

use of org.finra.herd.model.dto.DownloaderOutputManifestDto in project herd by FINRAOS.

the class DownloaderController method performDownload.

/**
 * Executes the downloader workflow.
 *
 * @param regServerAccessParamsDto the DTO for the parameters required to communicate with the herd registration server
 * @param manifestPath the local path to the manifest file
 * @param s3FileTransferRequestParamsDto the S3 file transfer DTO request parameters
 *
 * @throws InterruptedException if the upload thread was interrupted.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws IOException if an I/O error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = { "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" }, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.")
public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto) throws InterruptedException, JAXBException, IOException, URISyntaxException {
    boolean cleanUpTargetLocalDirectoryOnFailure = false;
    File targetLocalDirectory = null;
    try {
        // Process manifest file.
        DownloaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
        String storageName = getStorageNameFromManifest(manifest);
        // Get business object data from the herd registration server.
        downloaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
        BusinessObjectData businessObjectData = downloaderWebClient.getBusinessObjectData(manifest);
        manifest.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
        manifest.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
        s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() {

            @Override
            public AwsCredential getNewAwsCredential() throws Exception {
                return downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName).getAwsCredential();
            }
        });
        // Get a storage unit that belongs to the S3 storage.
        StorageUnit storageUnit = businessObjectDataHelper.getStorageUnitByStorageName(businessObjectData, storageName);
        // Get the expected S3 key prefix and S3 bucket name.
        S3KeyPrefixInformation s3KeyPrefixInformation = downloaderWebClient.getS3KeyPrefix(businessObjectData);
        // Check if the target folder (local directory + S3 key prefix) exists and try to create it if it does not.
        targetLocalDirectory = Paths.get(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix()).toFile();
        if (!targetLocalDirectory.isDirectory()) {
            // Create the local directory including any necessary but nonexistent parent directories.
            if (!targetLocalDirectory.mkdirs()) {
                throw new IllegalArgumentException(String.format("Failed to create target local directory \"%s\".", targetLocalDirectory.getPath()));
            }
        } else {
            // Check if the target local directory is empty.
            if (targetLocalDirectory.list().length > 0) {
                throw new IllegalArgumentException(String.format("The target local directory \"%s\" is not empty.", targetLocalDirectory.getPath()));
            }
        }
        // Get S3 bucket information.
        Storage storage = downloaderWebClient.getStorage(storageName);
        // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
        String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
        // Get the list of S3 files matching the expected S3 key prefix.
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixInformation.getS3KeyPrefix() + "/");
        // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
        List<String> actualS3Files = storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Service.listDirectory(s3FileTransferRequestParamsDto, true));
        // Validate S3 files before we start the download.
        storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, s3KeyPrefixInformation.getS3KeyPrefix());
        // Special handling for the maxThreads command line option.
        s3FileTransferRequestParamsDto.setMaxThreads(adjustIntegerValue(s3FileTransferRequestParamsDto.getMaxThreads(), MIN_THREADS, MAX_THREADS));
        // Download S3 files to the target local directory.
        s3FileTransferRequestParamsDto.setRecursive(true);
        cleanUpTargetLocalDirectoryOnFailure = true;
        s3Service.downloadDirectory(s3FileTransferRequestParamsDto);
        // Validate the downloaded files.
        storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit);
        // Log a list of files downloaded to the target local directory.
        if (LOGGER.isInfoEnabled()) {
            logLocalDirectoryContents(targetLocalDirectory);
        }
        // Create a downloader output manifest file.
        DownloaderOutputManifestDto downloaderOutputManifestDto = createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefixInformation.getS3KeyPrefix());
        manifestWriter.writeJsonManifest(targetLocalDirectory, OUTPUT_MANIFEST_FILE_NAME, downloaderOutputManifestDto);
    } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) {
        // occurred, let's rollback the data transfer by cleaning up the local target directory.
        if (cleanUpTargetLocalDirectoryOnFailure) {
            LOGGER.info(String.format("Rolling back the S3 data transfer by cleaning up \"%s\" target local directory.", targetLocalDirectory));
            HerdFileUtils.cleanDirectoryIgnoreException(targetLocalDirectory);
        }
        throw e;
    }
}