Examples with BusinessObjectDataKey org.finra.herd.model.api.xml.BusinessObjectDataKey used on opensource projects

Example 36 with BusinessObjectDataKey

use of org.finra.herd.model.api.xml.BusinessObjectDataKey in project herd by FINRAOS.

the class UploadDownloadHelperServiceImplTest method testPrepareForFileMove.

@Test
public void testPrepareForFileMove() {
    // Create an object key.
    String objectKey = UUID_VALUE;
    // Create a complete upload single parameters DTO.
    CompleteUploadSingleParamsDto completeUploadSingleParamsDto = new CompleteUploadSingleParamsDto();
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.UPLOADING);
    // Create a source storage entity.
    StorageEntity sourceStorageEntity = new StorageEntity();
    // Create a source business object data key.
    BusinessObjectDataKey sourceBusinessObjectDataKey = new BusinessObjectDataKey();
    sourceBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    // Create a source business object data entity.
    BusinessObjectDataEntity sourceBusinessObjectDataEntity = new BusinessObjectDataEntity();
    sourceBusinessObjectDataEntity.setId(ID);
    sourceBusinessObjectDataEntity.setPartitionValue(objectKey);
    sourceBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a list of source storage files.
    List<StorageFileEntity> sourceStorageFileEntities = new ArrayList<>();
    // Create a source storage unit.
    StorageUnitEntity sourceStorageUnitEntity = new StorageUnitEntity();
    sourceStorageUnitEntity.setBusinessObjectData(sourceBusinessObjectDataEntity);
    sourceStorageUnitEntity.setStorage(sourceStorageEntity);
    sourceStorageUnitEntity.setStorageFiles(sourceStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity sourceStorageFileEntity = new StorageFileEntity();
    sourceStorageFileEntities.add(sourceStorageFileEntity);
    sourceStorageFileEntity.setStorageUnit(sourceStorageUnitEntity);
    sourceStorageFileEntity.setPath(S3_KEY);
    sourceStorageFileEntity.setFileSizeBytes(FILE_SIZE);
    // Create a target storage entity.
    StorageEntity targetStorageEntity = new StorageEntity();
    // Create a target business object data key.
    BusinessObjectDataKey targetBusinessObjectDataKey = new BusinessObjectDataKey();
    targetBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE_2);
    // Create a list of source storage files.
    List<StorageFileEntity> targetStorageFileEntities = new ArrayList<>();
    // Create a target storage unit.
    StorageUnitEntity targetStorageUnitEntity = new StorageUnitEntity();
    targetStorageUnitEntity.setStorage(targetStorageEntity);
    targetStorageUnitEntity.setStorageFiles(targetStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity targetStorageFileEntity = new StorageFileEntity();
    targetStorageFileEntities.add(targetStorageFileEntity);
    targetStorageFileEntity.setPath(S3_KEY_2);
    // Create a target business object data entity.
    BusinessObjectDataEntity targetBusinessObjectDataEntity = new BusinessObjectDataEntity();
    targetBusinessObjectDataEntity.setId(ID_2);
    targetBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    targetBusinessObjectDataEntity.setStorageUnits(Collections.singletonList(targetStorageUnitEntity));
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Mock the external calls.
    when(storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey)).thenReturn(sourceStorageFileEntity);
    when(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity)).thenReturn(sourceBusinessObjectDataKey);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesByPartitionValue(objectKey)).thenReturn(Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity));
    when(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity)).thenReturn(targetBusinessObjectDataKey);
    when(storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE)).thenReturn(sourceStorageEntity);
    when(storageHelper.getStorageBucketName(sourceStorageEntity)).thenReturn(S3_BUCKET_NAME);
    when(storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity)).thenReturn(sourceStorageUnitEntity);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(storageHelper.getStorageBucketName(targetStorageEntity)).thenReturn(S3_BUCKET_NAME_2);
    when(storageHelper.getStorageKmsKeyId(targetStorageEntity)).thenReturn(AWS_KMS_KEY_ID);
    // Call the method under test.
    uploadDownloadHelperService.prepareForFileMoveImpl(objectKey, completeUploadSingleParamsDto);
    // Verify the external calls.
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(sourceBusinessObjectDataEntity);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesByPartitionValue(objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(targetBusinessObjectDataEntity);
    verify(storageDaoHelper).getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    verify(storageHelper).getStorageBucketName(sourceStorageEntity);
    verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
    verify(awsHelper, times(2)).getAwsParamsDto();
    verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE));
    verify(storageHelper).getStorageBucketName(targetStorageEntity);
    verify(storageHelper).getStorageKmsKeyId(targetStorageEntity);
    verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class));
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(notificationEventService).processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, sourceBusinessObjectDataKey, BusinessObjectDataStatusEntity.RE_ENCRYPTING, BusinessObjectDataStatusEntity.UPLOADING);
    verify(notificationEventService).processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, targetBusinessObjectDataKey, BusinessObjectDataStatusEntity.RE_ENCRYPTING, BusinessObjectDataStatusEntity.UPLOADING);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new CompleteUploadSingleParamsDto(sourceBusinessObjectDataKey, S3_BUCKET_NAME, S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, targetBusinessObjectDataKey, S3_BUCKET_NAME_2, S3_KEY_2, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, AWS_KMS_KEY_ID, awsParamsDto), completeUploadSingleParamsDto);
}

Example 37 with BusinessObjectDataKey

use of org.finra.herd.model.api.xml.BusinessObjectDataKey in project herd by FINRAOS.

the class UploadDownloadHelperServiceImplTest method testPrepareForFileMoveImplOptimisticLockException.

@Test
public void testPrepareForFileMoveImplOptimisticLockException() {
    // Create an object key.
    String objectKey = UUID_VALUE;
    // Create a complete upload single parameters DTO.
    CompleteUploadSingleParamsDto completeUploadSingleParamsDto = new CompleteUploadSingleParamsDto();
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.UPLOADING);
    // Create a source storage entity.
    StorageEntity sourceStorageEntity = new StorageEntity();
    // Create a source business object data key.
    BusinessObjectDataKey sourceBusinessObjectDataKey = new BusinessObjectDataKey();
    sourceBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    // Create a source business object data entity.
    BusinessObjectDataEntity sourceBusinessObjectDataEntity = new BusinessObjectDataEntity();
    sourceBusinessObjectDataEntity.setId(ID);
    sourceBusinessObjectDataEntity.setPartitionValue(objectKey);
    sourceBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a list of source storage files.
    List<StorageFileEntity> sourceStorageFileEntities = new ArrayList<>();
    // Create a source storage unit.
    StorageUnitEntity sourceStorageUnitEntity = new StorageUnitEntity();
    sourceStorageUnitEntity.setBusinessObjectData(sourceBusinessObjectDataEntity);
    sourceStorageUnitEntity.setStorage(sourceStorageEntity);
    sourceStorageUnitEntity.setStorageFiles(sourceStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity sourceStorageFileEntity = new StorageFileEntity();
    sourceStorageFileEntities.add(sourceStorageFileEntity);
    sourceStorageFileEntity.setStorageUnit(sourceStorageUnitEntity);
    sourceStorageFileEntity.setPath(S3_KEY);
    sourceStorageFileEntity.setFileSizeBytes(FILE_SIZE);
    // Create a target storage entity.
    StorageEntity targetStorageEntity = new StorageEntity();
    // Create a target business object data key.
    BusinessObjectDataKey targetBusinessObjectDataKey = new BusinessObjectDataKey();
    targetBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE_2);
    // Create a list of source storage files.
    List<StorageFileEntity> targetStorageFileEntities = new ArrayList<>();
    // Create a target storage unit.
    StorageUnitEntity targetStorageUnitEntity = new StorageUnitEntity();
    targetStorageUnitEntity.setStorage(targetStorageEntity);
    targetStorageUnitEntity.setStorageFiles(targetStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity targetStorageFileEntity = new StorageFileEntity();
    targetStorageFileEntities.add(targetStorageFileEntity);
    targetStorageFileEntity.setPath(S3_KEY_2);
    // Create a target business object data entity.
    BusinessObjectDataEntity targetBusinessObjectDataEntity = new BusinessObjectDataEntity();
    targetBusinessObjectDataEntity.setId(ID_2);
    targetBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    targetBusinessObjectDataEntity.setStorageUnits(Collections.singletonList(targetStorageUnitEntity));
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Mock the external calls.
    when(storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey)).thenReturn(sourceStorageFileEntity);
    when(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity)).thenReturn(sourceBusinessObjectDataKey);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesByPartitionValue(objectKey)).thenReturn(Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity));
    when(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity)).thenReturn(targetBusinessObjectDataKey);
    when(storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE)).thenReturn(sourceStorageEntity);
    when(storageHelper.getStorageBucketName(sourceStorageEntity)).thenReturn(S3_BUCKET_NAME);
    when(storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity)).thenReturn(sourceStorageUnitEntity);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(storageHelper.getStorageBucketName(targetStorageEntity)).thenReturn(S3_BUCKET_NAME_2);
    when(storageHelper.getStorageKmsKeyId(targetStorageEntity)).thenReturn(AWS_KMS_KEY_ID);
    doThrow(new OptimisticLockException(ERROR_MESSAGE)).when(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    when(jsonHelper.objectToJson(sourceBusinessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING);
    when(jsonHelper.objectToJson(targetBusinessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING_2);
    // Try to call the method under test.
    try {
        uploadDownloadHelperService.prepareForFileMoveImpl(objectKey, completeUploadSingleParamsDto);
    } catch (OptimisticLockException e) {
        assertEquals(String.format("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey=%s targetBusinessObjectDataKey=%s", BUSINESS_OBJECT_DATA_KEY_AS_STRING, BUSINESS_OBJECT_DATA_KEY_AS_STRING_2), e.getMessage());
    }
    // Verify the external calls.
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(sourceBusinessObjectDataEntity);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesByPartitionValue(objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(targetBusinessObjectDataEntity);
    verify(storageDaoHelper).getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    verify(storageHelper).getStorageBucketName(sourceStorageEntity);
    verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
    verify(awsHelper, times(2)).getAwsParamsDto();
    verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE));
    verify(storageHelper).getStorageBucketName(targetStorageEntity);
    verify(storageHelper).getStorageKmsKeyId(targetStorageEntity);
    verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class));
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(jsonHelper).objectToJson(sourceBusinessObjectDataKey);
    verify(jsonHelper).objectToJson(targetBusinessObjectDataKey);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new CompleteUploadSingleParamsDto(sourceBusinessObjectDataKey, S3_BUCKET_NAME, S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, NO_BDATA_STATUS, targetBusinessObjectDataKey, S3_BUCKET_NAME_2, S3_KEY_2, BusinessObjectDataStatusEntity.UPLOADING, NO_BDATA_STATUS, AWS_KMS_KEY_ID, awsParamsDto), completeUploadSingleParamsDto);
}

Example 38 with BusinessObjectDataKey

use of org.finra.herd.model.api.xml.BusinessObjectDataKey in project herd by FINRAOS.

the class AbstractDownloaderTest method uploadAndRegisterTestData.

/**
 * Uploads and registers a version if of the test business object data with the specified data files.
 *
 * @param s3KeyPrefix the destination S3 key prefix that must comply with the S3 naming conventions including the expected data version value
 * @param manifestFiles the test data files to be uploaded to S3 and registered
 * @param directoryPaths the list of directory paths to be created in S3 relative to the S3 key prefix
 */
protected void uploadAndRegisterTestData(String s3KeyPrefix, List<ManifestFile> manifestFiles, List<String> directoryPaths) throws Exception {
    uploadTestDataFilesToS3(s3KeyPrefix, manifestFiles, directoryPaths);
    UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto();
    uploaderInputManifestDto.setManifestFiles(manifestFiles);
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/");
    BusinessObjectData businessObjectData = downloaderWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, false);
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
    downloaderWebClient.addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, s3FileTransferRequestParamsDto, StorageEntity.MANAGED_STORAGE);
    downloaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    // Clean up the local input directory used for the test data files upload.
    FileUtils.cleanDirectory(LOCAL_TEMP_PATH_INPUT.toFile());
}

Example 39 with BusinessObjectDataKey

use of org.finra.herd.model.api.xml.BusinessObjectDataKey in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3FilesUnexpectedS3FileFound.

@Test
public void testValidateRegisteredS3FilesUnexpectedS3FileFound() throws IOException {
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Try to validate S3 files when unexpected S3 file exists.
    try {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating registered S3 files. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}

Example 40 with BusinessObjectDataKey

use of org.finra.herd.model.api.xml.BusinessObjectDataKey in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3FilesActualFileSizeMismatch.

@Test
public void testValidateRegisteredS3FilesActualFileSizeMismatch() throws IOException {
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));
    // Try to validate S3 files when expected expected file size does not match actual file size.
    try {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Specified file size of %d bytes for registered \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.", FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}