Examples with ManifestFile org.finra.herd.model.dto.ManifestFile used on opensource projects

Example 6 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderManifestReaderTest method testReadJsonManifestNoRowCount.

@Test
public void testReadJsonManifestNoRowCount() throws IOException {
    // Get an instance of uploader input manifest object.
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    // Make all the row counts null.
    for (ManifestFile manifestFile : uploaderInputManifestDto.getManifestFiles()) {
        manifestFile.setRowCount(null);
    }
    // Create and read a uploaderInputManifestDto file.
    UploaderInputManifestDto resultManifest = uploaderManifestReader.readJsonManifest(createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto));
    // Validate the results.
    assertUploaderManifest(uploaderInputManifestDto, resultManifest);
}

Example 7 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderController method getValidatedLocalFiles.

/**
 * Returns the list of File objects created from the specified list of local files after they are validated for existence and read access.
 *
 * @param localDir the local path to directory to be used to construct the relative absolute paths for the files to be validated
 * @param manifestFiles the list of manifest files that contain file paths relative to <code>localDir</code> to be validated.
 *
 * @return the list of validated File objects
 * @throws IllegalArgumentException if <code>localDir</code> or local files are not valid
 * @throws IOException if there is a filesystem query issue to construct a canonical form of an abstract file path
 */
private List<File> getValidatedLocalFiles(String localDir, List<ManifestFile> manifestFiles) throws IllegalArgumentException, IOException {
    // Create a "directory" file and ensure it is valid.
    File directory = new File(localDir);
    if (!directory.isDirectory() || !directory.canRead()) {
        throw new IllegalArgumentException(String.format("Invalid local base directory: %s", directory.getAbsolutePath()));
    }
    // For each file path from the list, create Java "File" objects to the real file location (i.e. with the directory
    // prepended to it), and verify that the file exists. If not, an IllegalArgumentException will be thrown.
    String basedir = directory.getAbsolutePath();
    List<File> resultFiles = new ArrayList<>();
    for (ManifestFile manifestFile : manifestFiles) {
        // Create a "real file" that points to the actual file on the file system (i.e. directory + manifest file path).
        String realFullPathFileName = Paths.get(basedir, manifestFile.getFileName()).toFile().getPath();
        realFullPathFileName = realFullPathFileName.replaceAll("\\\\", "/");
        File realFile = new File(realFullPathFileName);
        // Verify that the file exists and is readable.
        HerdFileUtils.verifyFileExistsAndReadable(realFile);
        // Verify that the name of the actual file on the file system exactly matches the name of the real file on the file system.
        // This will handle potential case issues on Windows systems. Note that the canonical file gives the actual file name on the system.
        // The non-canonical file gives the name as it was specified in the manifest.
        String realFileName = realFile.getName();
        String manifestFileName = realFile.getCanonicalFile().getName();
        if (!realFileName.equals(manifestFileName)) {
            throw new IllegalArgumentException("Manifest filename \"" + manifestFileName + "\" does not match actual filename \"" + realFileName + "\".");
        }
        resultFiles.add(realFile);
    }
    return resultFiles;
}

Example 8 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderManifestReader method validateManifest.

/**
 * Validates the manifest file.
 *
 * @param manifest the manifest to validate.
 *
 * @throws IllegalArgumentException if the manifest is not valid.
 */
@Override
protected void validateManifest(UploaderInputManifestDto manifest) throws IllegalArgumentException {
    // Perform the base validation.
    super.validateManifest(manifest);
    Assert.hasText(manifest.getBusinessObjectFormatVersion(), "Manifest business object format version must be specified.");
    Assert.notEmpty(manifest.getManifestFiles(), "Manifest must contain at least 1 file.");
    // Ensure row counts are all positive.
    for (ManifestFile manifestFile : manifest.getManifestFiles()) {
        Assert.hasText(manifestFile.getFileName(), "Manifest file can not have a blank filename.");
        // Trim to ensure we don't get errors with leading or trailing spaces causing "path" errors when validating that files exist.
        manifestFile.setFileName(manifestFile.getFileName().trim());
        if (manifestFile.getRowCount() != null) {
            Assert.isTrue(manifestFile.getRowCount() >= 0, "Manifest file \"" + manifestFile.getFileName() + "\" can not have a negative row count.");
        }
    }
    if (manifest.getAttributes() != null) {
        HashSet<String> attributeNameValidationSet = new HashSet<>();
        for (Map.Entry<String, String> entry : manifest.getAttributes().entrySet()) {
            String attributeName = entry.getKey().trim();
            Assert.hasText(attributeName, "Manifest attribute name must be specified.");
            // Validate attribute key length.
            Assert.isTrue(attributeName.length() <= MAX_ATTRIBUTE_NAME_LENGTH, String.format("Manifest attribute name is longer than %d characters.", MAX_ATTRIBUTE_NAME_LENGTH));
            // Ensure the attribute name isn't a duplicate by using a map with a "lowercase" name as the key for case insensitivity.
            String lowercaseAttributeName = attributeName.toLowerCase();
            Assert.isTrue(!attributeNameValidationSet.contains(lowercaseAttributeName), String.format("Duplicate manifest attribute name found: %s", attributeName));
            // Validate attribute value length.
            if (entry.getValue() != null) {
                Assert.isTrue(entry.getValue().length() <= MAX_ATTRIBUTE_VALUE_LENGTH, String.format("Manifest attribute value is longer than %d characters.", MAX_ATTRIBUTE_VALUE_LENGTH));
            }
            attributeNameValidationSet.add(lowercaseAttributeName);
        }
    }
    if (manifest.getBusinessObjectDataParents() != null) {
        HashSet<BusinessObjectDataKey> parentValidationSet = new HashSet<>();
        for (BusinessObjectDataKey businessObjectDataKey : manifest.getBusinessObjectDataParents()) {
            // Perform validation.
            Assert.hasText(businessObjectDataKey.getBusinessObjectDefinitionName(), "Manifest parent business object definition name must be specified.");
            Assert.hasText(businessObjectDataKey.getBusinessObjectFormatUsage(), "Manifest parent business object format usage must be specified.");
            Assert.hasText(businessObjectDataKey.getBusinessObjectFormatFileType(), "Manifest parent business object format file type must be specified.");
            Assert.hasText(businessObjectDataKey.getPartitionValue(), "Manifest parent business object data partition value must be specified.");
            // Remove leading and trailing spaces.
            businessObjectDataKey.setBusinessObjectDefinitionName(businessObjectDataKey.getBusinessObjectDefinitionName().trim());
            businessObjectDataKey.setBusinessObjectFormatUsage(businessObjectDataKey.getBusinessObjectFormatUsage().trim());
            businessObjectDataKey.setBusinessObjectFormatFileType(businessObjectDataKey.getBusinessObjectFormatFileType().trim());
            businessObjectDataKey.setPartitionValue(businessObjectDataKey.getPartitionValue().trim());
            // Ensure the business object data parent isn't a duplicate by using a unique set
            // with "lowercase" parent alternate key fields that are case insensitive.
            BusinessObjectDataKey lowercaseKey = new BusinessObjectDataKey();
            lowercaseKey.setBusinessObjectDefinitionName(businessObjectDataKey.getBusinessObjectDefinitionName().toLowerCase());
            lowercaseKey.setBusinessObjectFormatUsage(businessObjectDataKey.getBusinessObjectFormatUsage().toLowerCase());
            lowercaseKey.setBusinessObjectFormatFileType(businessObjectDataKey.getBusinessObjectFormatFileType().toLowerCase());
            lowercaseKey.setBusinessObjectFormatVersion(businessObjectDataKey.getBusinessObjectFormatVersion());
            lowercaseKey.setPartitionValue(businessObjectDataKey.getPartitionValue());
            lowercaseKey.setBusinessObjectDataVersion(businessObjectDataKey.getBusinessObjectDataVersion());
            Assert.isTrue(!parentValidationSet.contains(lowercaseKey), String.format("Duplicate manifest business object data parent found: {%s}", businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
            parentValidationSet.add(lowercaseKey);
        }
    }
}

Example 9 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadManifestContainsDuplicateFileNames.

@Test(expected = IllegalArgumentException.class)
public void testPerformUploadManifestContainsDuplicateFileNames() throws Exception {
    List<ManifestFile> duplicateTestDataFiles = new ArrayList<>();
    duplicateTestDataFiles.addAll(testManifestFiles);
    duplicateTestDataFiles.add(testManifestFiles.get(0));
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : testManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(duplicateTestDataFiles);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Try to upload business object data having duplicate file names.
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}

Example 10 with ManifestFile

use of org.finra.herd.model.dto.ManifestFile in project herd by FINRAOS.

the class UploaderControllerTest method testPerformUploadManifestFileNameDoesNotMatchActualFileName.

@Test(expected = IllegalArgumentException.class)
public void testPerformUploadManifestFileNameDoesNotMatchActualFileName() throws Exception {
    List<ManifestFile> declaredManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "test-data-2.txt"), FILE_SIZE_1_KB);
    List<ManifestFile> actualManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "TEST-DATA-2.TXT"), FILE_SIZE_1_KB);
    // Create local data files in LOCAL_TEMP_PATH_INPUT directory
    for (ManifestFile manifestFile : actualManifestFiles) {
        createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB);
    }
    // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory.
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(declaredManifestFiles);
    File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto);
    Assert.assertTrue(manifestFile.isFile());
    // Try to upload business object data with one of the file having name that does not match to incorrect name specified.
    RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true).withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
    uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS);
}