Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 41 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataStorageUnitRestControllerTest method testCreateBusinessObjectDataStorageUnit.

@Test
public void testCreateBusinessObjectDataStorageUnit() {
    // Create a business object data storage unit key.
    BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey = new BusinessObjectDataStorageUnitKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME);
    // Create a storage unit directory.
    StorageDirectory storageDirectory = new StorageDirectory(STORAGE_DIRECTORY_PATH);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Collections.singletonList(new StorageFile(FILE_NAME, FILE_SIZE, ROW_COUNT));
    // Create a business object data storage unit request.
    BusinessObjectDataStorageUnitCreateRequest request = new BusinessObjectDataStorageUnitCreateRequest(businessObjectDataStorageUnitKey, storageDirectory, storageFiles, DISCOVER_STORAGE_FILES);
    // Create a business object data storage unit response.
    BusinessObjectDataStorageUnitCreateResponse response = new BusinessObjectDataStorageUnitCreateResponse(businessObjectDataStorageUnitKey, storageDirectory, storageFiles);
    // Mock the external calls.
    when(businessObjectDataStorageUnitService.createBusinessObjectDataStorageUnit(request)).thenReturn(response);
    // Call the method under test.
    BusinessObjectDataStorageUnitCreateResponse result = businessObjectDataStorageUnitRestController.createBusinessObjectDataStorageUnit(request);
    // Verify the external calls.
    verify(businessObjectDataStorageUnitService).createBusinessObjectDataStorageUnit(request);
    verifyNoMoreInteractions(businessObjectDataStorageUnitService);
    // Validate the results.
    assertEquals(response, result);
}

Example 42 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method createFile.

private StorageFile createFile(String filePath, Long size, Long rowCount) {
    StorageFile f = new StorageFile();
    f.setFilePath(filePath);
    f.setFileSizeBytes(size);
    f.setRowCount(rowCount);
    return f;
}

Example 43 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateStorageFiles.

/**
 * Validates a list of StorageFiles against the expected values.
 *
 * @param expectedStorageFiles the list of expected StorageFiles
 * @param actualStorageFiles the list of actual StorageFiles to be validated
 */
public void validateStorageFiles(List<StorageFile> expectedStorageFiles, List<StorageFile> actualStorageFiles) {
    assertEquals(expectedStorageFiles.size(), actualStorageFiles.size());
    for (int i = 0; i < expectedStorageFiles.size(); i++) {
        StorageFile expectedStorageFile = expectedStorageFiles.get(i);
        StorageFile actualStorageFile = actualStorageFiles.get(i);
        assertEquals(expectedStorageFile.getFilePath(), actualStorageFile.getFilePath());
        assertEquals(expectedStorageFile.getFileSizeBytes(), actualStorageFile.getFileSizeBytes());
        assertEquals(expectedStorageFile.getRowCount(), actualStorageFile.getRowCount());
    }
}

Example 44 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.

/**
 * Validates business object data against specified arguments and expected (hard coded) test values.
 *
 * @param request the business object data create request
 * @param expectedBusinessObjectDataVersion the expected business object data version
 * @param expectedLatestVersion the expected business
 * @param actualBusinessObjectData the business object data availability object instance to be validated
 */
public void validateBusinessObjectData(BusinessObjectDataCreateRequest request, Integer expectedBusinessObjectDataVersion, Boolean expectedLatestVersion, BusinessObjectData actualBusinessObjectData) {
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(org.apache.commons.lang3.StringUtils.isNotBlank(request.getNamespace()) ? request.getNamespace() : AbstractServiceTest.NAMESPACE, request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()));
    List<String> expectedSubPartitionValues = CollectionUtils.isEmpty(request.getSubPartitionValues()) ? new ArrayList<>() : request.getSubPartitionValues();
    String expectedStatusCode = org.apache.commons.lang3.StringUtils.isNotBlank(request.getStatus()) ? request.getStatus() : BusinessObjectDataStatusEntity.VALID;
    StorageUnitCreateRequest storageUnitCreateRequest = request.getStorageUnits().get(0);
    StorageEntity storageEntity = storageDao.getStorageByName(storageUnitCreateRequest.getStorageName());
    String expectedStorageDirectoryPath = storageUnitCreateRequest.getStorageDirectory() != null ? storageUnitCreateRequest.getStorageDirectory().getDirectoryPath() : null;
    List<StorageFile> expectedStorageFiles = CollectionUtils.isEmpty(storageUnitCreateRequest.getStorageFiles()) ? null : storageUnitCreateRequest.getStorageFiles();
    List<Attribute> expectedAttributes = CollectionUtils.isEmpty(request.getAttributes()) ? new ArrayList<>() : request.getAttributes();
    validateBusinessObjectData(businessObjectFormatEntity, request.getPartitionValue(), expectedSubPartitionValues, expectedBusinessObjectDataVersion, expectedLatestVersion, expectedStatusCode, storageEntity.getName(), expectedStorageDirectoryPath, expectedStorageFiles, expectedAttributes, actualBusinessObjectData);
}

Example 45 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method createStorageFileEntitiesFromStorageFiles.

/**
 * Creates a list of storage file entities from a list of storage files.
 *
 * @param storageFiles the list of storage files
 * @param storageEntity the storage entity
 * @param storageFilesDiscovered specifies whether the storage files were actually discovered in the storage
 * @param expectedS3KeyPrefix the expected S3 key prefix
 * @param storageUnitEntity the storage unit entity that storage file entities will belong to
 * @param directoryPath the storage directory path
 * @param validatePathPrefix specifies whether the storage has S3 key prefix validation enabled
 * @param validateFileExistence specifies whether the storage has file existence enabled
 * @param validateFileSize specifies whether the storage has file validation enabled
 * @param isS3StoragePlatform specifies whether the storage platform type is S3
 *
 * @return the list of storage file entities
 */
private List<StorageFileEntity> createStorageFileEntitiesFromStorageFiles(List<StorageFile> storageFiles, StorageEntity storageEntity, boolean storageFilesDiscovered, String expectedS3KeyPrefix, StorageUnitEntity storageUnitEntity, String directoryPath, boolean validatePathPrefix, boolean validateFileExistence, boolean validateFileSize, boolean isS3StoragePlatform) {
    List<StorageFileEntity> storageFileEntities = null;
    // Process storage files if they are specified.
    if (CollectionUtils.isNotEmpty(storageFiles)) {
        storageFileEntities = new ArrayList<>();
        storageUnitEntity.setStorageFiles(storageFileEntities);
        // If the validate file existence flag is configured for this storage and storage files were not discovered, prepare for S3 file validation.
        S3FileTransferRequestParamsDto params = null;
        Map<String, StorageFile> actualS3Keys = null;
        if (validateFileExistence && isS3StoragePlatform && !storageFilesDiscovered) {
            // Get the validate file parameters.
            params = getFileValidationParams(storageEntity, expectedS3KeyPrefix, storageUnitEntity, validatePathPrefix);
            // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
            actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
        }
        // storage by some other business object data that start with the expected S3 key prefix.
        if (validatePathPrefix && isS3StoragePlatform) {
            // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
            String expectedS3KeyPrefixWithTrailingSlash = expectedS3KeyPrefix + "/";
            Long registeredStorageFileCount = storageFileDao.getStorageFileCount(storageEntity.getName(), expectedS3KeyPrefixWithTrailingSlash);
            if (registeredStorageFileCount > 0) {
                throw new AlreadyExistsException(String.format("Found %d storage file(s) matching \"%s\" S3 key prefix in \"%s\" " + "storage that is registered with another business object data.", registeredStorageFileCount, expectedS3KeyPrefix, storageEntity.getName()));
            }
        }
        for (StorageFile storageFile : storageFiles) {
            StorageFileEntity storageFileEntity = new StorageFileEntity();
            storageFileEntities.add(storageFileEntity);
            storageFileEntity.setStorageUnit(storageUnitEntity);
            storageFileEntity.setPath(storageFile.getFilePath());
            storageFileEntity.setFileSizeBytes(storageFile.getFileSizeBytes());
            storageFileEntity.setRowCount(storageFile.getRowCount());
            // Skip storage file validation if storage files were discovered.
            if (!storageFilesDiscovered) {
                // Otherwise, if a directory path is specified, ensure it is consistent with the file path.
                if (validatePathPrefix && isS3StoragePlatform) {
                    // Ensure the S3 file key prefix adheres to the S3 naming convention.
                    Assert.isTrue(storageFileEntity.getPath().startsWith(expectedS3KeyPrefix), String.format("Specified storage file path \"%s\" does not match the expected S3 key prefix \"%s\".", storageFileEntity.getPath(), expectedS3KeyPrefix));
                } else if (directoryPath != null) {
                    // When storage directory path is specified, ensure that storage file path starts with it.
                    Assert.isTrue(storageFileEntity.getPath().startsWith(directoryPath), String.format("Storage file path \"%s\" does not match the storage directory path \"%s\".", storageFileEntity.getPath(), directoryPath));
                }
                // Ensure the file exists in S3 if the validate file existence flag is configured for this storage.
                if (validateFileExistence && isS3StoragePlatform) {
                    // Validate storage file.
                    storageFileHelper.validateStorageFile(storageFile, params.getS3BucketName(), actualS3Keys, validateFileSize);
                }
            }
        }
    }
    return storageFileEntities;
}