Examples with ManifestFile org.finra.herd.model.dto.ManifestFile used on opensource projects

Example 1 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class DownloaderController method createDownloaderOutputManifestDto.

/**
 * Creates a downloader output manifest instance from the business object data.
 *
 * @param businessObjectData the business object data that we need to create the manifest for
 * @param storageUnit the S3 storage unit for this business object data
 * @param s3KeyPrefix the S3 key prefix for this business object data
 *
 * @return the created downloader output manifest instance
 */
protected DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix) {
    DownloaderOutputManifestDto downloaderOutputManifestDto = new DownloaderOutputManifestDto();
    // Populate basic fields.
    downloaderOutputManifestDto.setNamespace(businessObjectData.getNamespace());
    downloaderOutputManifestDto.setBusinessObjectDefinitionName(businessObjectData.getBusinessObjectDefinitionName());
    downloaderOutputManifestDto.setBusinessObjectFormatUsage(businessObjectData.getBusinessObjectFormatUsage());
    downloaderOutputManifestDto.setBusinessObjectFormatFileType(businessObjectData.getBusinessObjectFormatFileType());
    downloaderOutputManifestDto.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    downloaderOutputManifestDto.setPartitionKey(businessObjectData.getPartitionKey());
    downloaderOutputManifestDto.setPartitionValue(businessObjectData.getPartitionValue());
    downloaderOutputManifestDto.setSubPartitionValues(businessObjectData.getSubPartitionValues());
    downloaderOutputManifestDto.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
    downloaderOutputManifestDto.setStorageName(storageUnit.getStorage().getName());
    // Build a list of manifest files with paths relative to the S3 key prefix.
    List<ManifestFile> manifestFiles = new ArrayList<>();
    downloaderOutputManifestDto.setManifestFiles(manifestFiles);
    if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) {
        for (StorageFile storageFile : storageUnit.getStorageFiles()) {
            ManifestFile manifestFile = new ManifestFile();
            manifestFiles.add(manifestFile);
            manifestFile.setFileName(storageFile.getFilePath().replace(s3KeyPrefix, ""));
            manifestFile.setFileSizeBytes(storageFile.getFileSizeBytes());
            manifestFile.setRowCount(storageFile.getRowCount());
        }
    }
    // Populate the attributes.
    HashMap<String, String> attributes = new HashMap<>();
    if (!CollectionUtils.isEmpty(businessObjectData.getAttributes())) {
        for (Attribute attribute : businessObjectData.getAttributes()) {
            attributes.put(attribute.getName(), attribute.getValue());
        }
    }
    downloaderOutputManifestDto.setAttributes(attributes);
    // Populate the business object data parents and children.
    downloaderOutputManifestDto.setBusinessObjectDataParents(businessObjectData.getBusinessObjectDataParents());
    downloaderOutputManifestDto.setBusinessObjectDataChildren(businessObjectData.getBusinessObjectDataChildren());
    return downloaderOutputManifestDto;
}

Example 2 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class DataBridgeWebClient method addStorageFiles.

/**
 * Calls the registration server to add storage files to the business object data.
 *
 * @param businessObjectDataKey the business object data key
 * @param manifest the uploader input manifest file
 * @param s3FileTransferRequestParamsDto the S3 file transfer request parameters to be used to retrieve local path and S3 key prefix values
 * @param storageName the storage name
 *
 * @return the business object data create storage files response turned by the registration server.
 * @throws IOException if an I/O error was encountered.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = "VA_FORMAT_STRING_USES_NEWLINE", justification = "We will use the standard carriage return character.")
public BusinessObjectDataStorageFilesCreateResponse addStorageFiles(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, String storageName) throws IOException, JAXBException, URISyntaxException {
    LOGGER.info("Adding storage files to the business object data ...");
    BusinessObjectDataStorageFilesCreateRequest request = new BusinessObjectDataStorageFilesCreateRequest();
    request.setNamespace(businessObjectDataKey.getNamespace());
    request.setBusinessObjectDefinitionName(businessObjectDataKey.getBusinessObjectDefinitionName());
    request.setBusinessObjectFormatUsage(businessObjectDataKey.getBusinessObjectFormatUsage());
    request.setBusinessObjectFormatFileType(businessObjectDataKey.getBusinessObjectFormatFileType());
    request.setBusinessObjectFormatVersion(businessObjectDataKey.getBusinessObjectFormatVersion());
    request.setPartitionValue(businessObjectDataKey.getPartitionValue());
    request.setSubPartitionValues(businessObjectDataKey.getSubPartitionValues());
    request.setBusinessObjectDataVersion(businessObjectDataKey.getBusinessObjectDataVersion());
    request.setStorageName(storageName);
    List<StorageFile> storageFiles = new ArrayList<>();
    request.setStorageFiles(storageFiles);
    String localPath = s3FileTransferRequestParamsDto.getLocalPath();
    String s3KeyPrefix = s3FileTransferRequestParamsDto.getS3KeyPrefix();
    List<ManifestFile> localFiles = manifest.getManifestFiles();
    for (ManifestFile manifestFile : localFiles) {
        StorageFile storageFile = new StorageFile();
        storageFiles.add(storageFile);
        // Since the S3 key prefix represents a directory it is expected to contain a trailing '/' character.
        storageFile.setFilePath((s3KeyPrefix + manifestFile.getFileName()).replaceAll("\\\\", "/"));
        storageFile.setFileSizeBytes(Paths.get(localPath, manifestFile.getFileName()).toFile().length());
        storageFile.setRowCount(manifestFile.getRowCount());
    }
    // Create a JAXB context and marshaller
    JAXBContext requestContext = JAXBContext.newInstance(BusinessObjectDataStorageFilesCreateRequest.class);
    Marshaller requestMarshaller = requestContext.createMarshaller();
    requestMarshaller.setProperty(Marshaller.JAXB_ENCODING, StandardCharsets.UTF_8.name());
    requestMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
    StringWriter sw = new StringWriter();
    requestMarshaller.marshal(request, sw);
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse;
    try (CloseableHttpClient client = httpClientOperations.createHttpClient()) {
        URI uri = new URIBuilder().setScheme(getUriScheme()).setHost(regServerAccessParamsDto.getRegServerHost()).setPort(regServerAccessParamsDto.getRegServerPort()).setPath(HERD_APP_REST_URI_PREFIX + "/businessObjectDataStorageFiles").build();
        HttpPost post = new HttpPost(uri);
        post.addHeader("Content-Type", DEFAULT_CONTENT_TYPE);
        post.addHeader("Accepts", DEFAULT_ACCEPT);
        // If SSL is enabled, set the client authentication header.
        if (regServerAccessParamsDto.isUseSsl()) {
            post.addHeader(getAuthorizationHeader());
        }
        post.setEntity(new StringEntity(sw.toString()));
        LOGGER.info(String.format("    HTTP POST URI: %s", post.getURI().toString()));
        LOGGER.info(String.format("    HTTP POST Headers: %s", Arrays.toString(post.getAllHeaders())));
        LOGGER.info(String.format("    HTTP POST Entity Content:%n%s", sw.toString()));
        // getBusinessObjectDataStorageFilesCreateResponse() might return a null. That happens when the web client gets status code 200 back from
        // the service (add storage files is a success), but it fails to retrieve or deserialize the actual HTTP response.
        // Please note that processXmlHttpResponse() is responsible for logging the exception info as a warning.
        businessObjectDataStorageFilesCreateResponse = getBusinessObjectDataStorageFilesCreateResponse(httpClientOperations.execute(client, post));
    }
    LOGGER.info("Successfully added storage files to the registered business object data.");
    return businessObjectDataStorageFilesCreateResponse;
}

Example 3 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class AbstractDataBridgeTest method getManifestFilesFromFileNames.

/**
 * Gets a list of manifest files from a list of file names.
 *
 * @param fileNames the list of file names.
 * @param fileSizeBytes the file size in bytes for each manifest file.
 *
 * @return the list of manifest files.
 */
protected List<ManifestFile> getManifestFilesFromFileNames(List<String> fileNames, long fileSizeBytes) {
    List<ManifestFile> manifestFiles = new ArrayList<>();
    for (int i = 0; i < fileNames.size(); i++) {
        String fileName = fileNames.get(i);
        ManifestFile manifestFile = new ManifestFile();
        manifestFiles.add(manifestFile);
        manifestFile.setFileName(fileName);
        manifestFile.setRowCount((long) i);
        manifestFile.setFileSizeBytes(fileSizeBytes);
    }
    return manifestFiles;
}

Example 4 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadInvalidLocalDir.

@Test
public void testPerformUploadInvalidLocalDir() throws Exception {
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), getTestUploaderInputManifestDto());
    Assert.assertTrue(manifestFile.isFile());
    // Try to run upload by specifying a non-existing local directory.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setLocalPath(Paths.get(LOCAL_TEMP_PATH_INPUT.toString(), "I_DO_NOT_EXIST").toString());
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    try {
        uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
        fail("Should throw an IllegalArgumentException when local directory does not exist.");
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().startsWith("Invalid local base directory"));
    }
}

Example 5 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderControllerTest method runUpload.

/**
 * Runs a normal upload scenario.
 *
 * @param numOfThreads the maximum number of threads to use for file transfer to S3
 * @param attributes the attributes to be associated with the test data being uploaded
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created
 * @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
 * @param hostname optional override of the default web service hostname.
 * @param storageName optional storage name
 */
protected void runUpload(Integer numOfThreads, HashMap<String, String> attributes, Boolean createNewVersion, Boolean force, String hostname, String storageName) throws Exception {
    String hostnameToUse = hostname == null ? WEB_SERVICE_HOSTNAME : hostname;
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : testManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setAttributes(attributes);
    uploaderInputManifestDto.setStorageName(storageName);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Perform the upload.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_INPUT.toString());
    s3FileTransferRequestParamsDto.setMaxThreads(numOfThreads);
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(hostnameToUse).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, createNewVersion, force, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}