Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 66 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateCopiedS3Files.

@Test
public void testValidateCopiedS3Files() throws IOException {
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES) {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }
    // Validate the files.
    storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
}

Example 67 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateCopiedS3FilesUnexpectedS3FileFound.

@Test
public void testValidateCopiedS3FilesUnexpectedS3FileFound() throws IOException {
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Try to validate S3 files when unexpected S3 file exists.
    try {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating copied S3 files. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}

Example 68 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateCopiedS3FilesActualFileSizeMismatch.

@Test
public void testValidateCopiedS3FilesActualFileSizeMismatch() throws IOException {
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));
    // Try to validate S3 files when expected file size does not match actual file size.
    try {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Specified file size of %d bytes for copied \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.", FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}

Example 69 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateCopiedS3FilesActualFileNoExists.

@Test
public void testValidateCopiedS3FilesActualFileNoExists() throws IOException {
    // Create two lists of expected and actual storage files, with one expected file not being added to the list of actual files.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, ROW_COUNT_1000));
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    // Try to validate S3 files when expected S3 file does not exist.
    try {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an ObjectNotFoundException when the copied S3 file does not exist.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Copied file \"%s\" does not exist in \"%s\" storage.", TARGET_S3_KEY, STORAGE_NAME), e.getMessage());
    }
}

Example 70 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StoragePolicyProcessorJmsMessageListenerTest method testProcessMessage.

@Test
public void testProcessMessage() throws Exception {
    // Build the expected S3 key prefix for test business object data.
    String s3KeyPrefix = getExpectedS3KeyPrefix(BDEF_NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, null, null, DATA_VERSION);
    // Create S3FileTransferRequestParamsDto to access the source S3 bucket location.
    // Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto sourceS3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(s3KeyPrefix + "/").build();
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting(STORAGE_POLICY_NAMESPACE_CD, Arrays.asList(STORAGE_POLICY_RULE_TYPE), BDEF_NAMESPACE, BDEF_NAME, Arrays.asList(FORMAT_FILE_TYPE_CODE), Arrays.asList(STORAGE_NAME), Arrays.asList(StoragePolicyTransitionTypeEntity.GLACIER));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create and persist an ENABLED storage unit in the source storage.
    StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Add storage files to the source storage unit.
    for (String filePath : LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(sourceStorageUnitEntity, s3KeyPrefix + "/" + filePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    // Get the source storage files.
    List<StorageFile> sourceStorageFiles = storageFileHelper.createStorageFilesFromEntities(sourceStorageUnitEntity.getStorageFiles());
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Override configuration to specify some settings required for testing.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN.getKey(), S3_OBJECT_TAGGER_ROLE_ARN);
    overrideMap.put(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME.getKey(), S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Put relative S3 files into the source S3 bucket.
        for (StorageFile storageFile : sourceStorageFiles) {
            s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFile.getFilePath(), new ByteArrayInputStream(new byte[storageFile.getFileSizeBytes().intValue()]), null), null);
        }
        // Perform a storage policy transition.
        storagePolicyProcessorJmsMessageListener.processMessage(jsonHelper.objectToJson(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, INITIAL_VERSION)), null);
        // Validate the status of the source storage unit.
        assertEquals(StorageUnitStatusEntity.ARCHIVED, sourceStorageUnitEntity.getStatus().getCode());
    } finally {
        // Delete test files from S3 storage.
        if (!s3Dao.listDirectory(sourceS3FileTransferRequestParamsDto).isEmpty()) {
            s3Dao.deleteDirectory(sourceS3FileTransferRequestParamsDto);
        }
        s3Operations.rollback();
        // Restore the property sources so we don't affect other tests.
        restorePropertySourceInEnvironment();
    }
}