Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 36 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method testCreateBusinessObjectDataStorageFilesAutoDiscovery.

@Test
public void testCreateBusinessObjectDataStorageFilesAutoDiscovery() throws Exception {
    createData(testS3KeyPrefix, true, Arrays.asList(testS3KeyPrefix + "/" + FILE_PATH_1));
    businessObjectDataServiceTestHelper.prepareTestS3Files(testS3KeyPrefix, localTempPath, Arrays.asList(FILE_PATH_1, FILE_PATH_2));
    // Discover storage files in S3 managed storage.
    BusinessObjectDataStorageFilesCreateResponse response = businessObjectDataStorageFileService.createBusinessObjectDataStorageFiles(new BusinessObjectDataStorageFilesCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, StorageEntity.MANAGED_STORAGE, NO_STORAGE_FILES, DISCOVER_STORAGE_FILES));
    // Validate the returned object.
    assertEquals(new BusinessObjectDataStorageFilesCreateResponse(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION, StorageEntity.MANAGED_STORAGE, Arrays.asList(new StorageFile(testS3KeyPrefix + "/" + FILE_PATH_2, FILE_SIZE_1_KB, NO_ROW_COUNT))), response);
}

Example 37 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getNewBusinessObjectDataCreateRequest.

/**
 * Gets a newly created business object data create request.
 *
 * @param includeAttributes If true, attribute definitions and attributes will be included. Otherwise, not.
 *
 * @return the business object create request.
 */
public BusinessObjectDataCreateRequest getNewBusinessObjectDataCreateRequest(boolean includeAttributes) {
    // Crete a test business object format (and associated data).
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(includeAttributes);
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity();
    // Create a request to create business object data.
    BusinessObjectDataCreateRequest businessObjectDataCreateRequest = new BusinessObjectDataCreateRequest();
    businessObjectDataCreateRequest.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
    businessObjectDataCreateRequest.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
    businessObjectDataCreateRequest.setBusinessObjectFormatUsage(businessObjectFormatEntity.getUsage());
    businessObjectDataCreateRequest.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
    businessObjectDataCreateRequest.setBusinessObjectFormatVersion(businessObjectFormatEntity.getBusinessObjectFormatVersion());
    businessObjectDataCreateRequest.setPartitionKey(businessObjectFormatEntity.getPartitionKey());
    businessObjectDataCreateRequest.setPartitionValue(AbstractServiceTest.PARTITION_VALUE);
    businessObjectDataCreateRequest.setSubPartitionValues(AbstractServiceTest.SUBPARTITION_VALUES);
    List<StorageUnitCreateRequest> storageUnits = new ArrayList<>();
    businessObjectDataCreateRequest.setStorageUnits(storageUnits);
    StorageUnitCreateRequest storageUnit = new StorageUnitCreateRequest();
    storageUnits.add(storageUnit);
    storageUnit.setStorageName(storageEntity.getName());
    StorageDirectory storageDirectory = new StorageDirectory();
    storageUnit.setStorageDirectory(storageDirectory);
    storageDirectory.setDirectoryPath("Folder");
    List<StorageFile> storageFiles = new ArrayList<>();
    storageUnit.setStorageFiles(storageFiles);
    StorageFile storageFile1 = new StorageFile();
    storageFiles.add(storageFile1);
    storageFile1.setFilePath("Folder/file1.gz");
    storageFile1.setFileSizeBytes(0L);
    storageFile1.setRowCount(0L);
    StorageFile storageFile2 = new StorageFile();
    storageFiles.add(storageFile2);
    storageFile2.setFilePath("Folder/file2.gz");
    storageFile2.setFileSizeBytes(2999L);
    storageFile2.setRowCount(1000L);
    StorageFile storageFile3 = new StorageFile();
    storageFiles.add(storageFile3);
    storageFile3.setFilePath("Folder/file3.gz");
    storageFile3.setFileSizeBytes(Long.MAX_VALUE);
    storageFile3.setRowCount(Long.MAX_VALUE);
    if (includeAttributes) {
        businessObjectDataCreateRequest.setAttributes(businessObjectDefinitionServiceTestHelper.getNewAttributes());
    }
    List<BusinessObjectDataKey> businessObjectDataParents = new ArrayList<>();
    businessObjectDataCreateRequest.setBusinessObjectDataParents(businessObjectDataParents);
    // Create 2 parents.
    for (int i = 0; i < 2; i++) {
        BusinessObjectDataEntity parentBusinessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity();
        BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey();
        businessObjectDataKey.setNamespace(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getNamespace().getCode());
        businessObjectDataKey.setBusinessObjectDefinitionName(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getName());
        businessObjectDataKey.setBusinessObjectFormatUsage(parentBusinessObjectDataEntity.getBusinessObjectFormat().getUsage());
        businessObjectDataKey.setBusinessObjectFormatFileType(parentBusinessObjectDataEntity.getBusinessObjectFormat().getFileType().getCode());
        businessObjectDataKey.setBusinessObjectFormatVersion(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectFormatVersion());
        businessObjectDataKey.setPartitionValue(parentBusinessObjectDataEntity.getPartitionValue());
        businessObjectDataKey.setBusinessObjectDataVersion(parentBusinessObjectDataEntity.getVersion());
        businessObjectDataKey.setSubPartitionValues(businessObjectDataHelper.getSubPartitionValues(parentBusinessObjectDataEntity));
        businessObjectDataParents.add(businessObjectDataKey);
    }
    return businessObjectDataCreateRequest;
}

Example 38 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.

public void validateBusinessObjectData(String expectedNamespaceCode, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, String expectedBusinessObjectDataStatus, List<Attribute> expectedAttributes, String expectedStorageName, String expectedFileName, Long expectedFileSizeBytes, BusinessObjectData businessObjectData) {
    assertNotNull(businessObjectData);
    // Validate business object data alternate key values.
    assertEquals(expectedNamespaceCode, businessObjectData.getNamespace());
    assertEquals(expectedBusinessObjectDefinitionName, businessObjectData.getBusinessObjectDefinitionName());
    assertEquals(expectedBusinessObjectFormatUsage, businessObjectData.getBusinessObjectFormatUsage());
    assertEquals(expectedBusinessObjectFormatFileType, businessObjectData.getBusinessObjectFormatFileType());
    assertEquals(expectedBusinessObjectFormatVersion, Integer.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    // The business object data partition value must contain an UUID value.
    assertNotNull(businessObjectData.getPartitionValue());
    assertEquals(AbstractServiceTest.EXPECTED_UUID_SIZE, businessObjectData.getPartitionValue().length());
    assertEquals(AbstractServiceTest.NO_SUBPARTITION_VALUES, businessObjectData.getSubPartitionValues());
    assertEquals(AbstractServiceTest.INITIAL_DATA_VERSION, Integer.valueOf(businessObjectData.getVersion()));
    // Validate business object data status.
    assertTrue(businessObjectData.isLatestVersion());
    assertEquals(expectedBusinessObjectDataStatus, businessObjectData.getStatus());
    // Validate business object data attributes.
    businessObjectDefinitionServiceTestHelper.validateAttributes(expectedAttributes, businessObjectData.getAttributes());
    // Validate storage unit contents.
    assertEquals(1, businessObjectData.getStorageUnits().size());
    StorageUnit storageUnit = businessObjectData.getStorageUnits().get(0);
    assertEquals(expectedStorageName, storageUnit.getStorage().getName());
    String expectedStorageDirectoryPath = String.format("%s/%s/%s", AbstractServiceTest.ENVIRONMENT_NAME.trim().toLowerCase().replace('_', '-'), expectedNamespaceCode.trim().toLowerCase().replace('_', '-'), businessObjectData.getPartitionValue());
    assertEquals(expectedStorageDirectoryPath, storageUnit.getStorageDirectory().getDirectoryPath());
    assertEquals(1, storageUnit.getStorageFiles().size());
    StorageFile storageFile = storageUnit.getStorageFiles().get(0);
    String expectedStorageFilePath = String.format("%s/%s", expectedStorageDirectoryPath, expectedFileName);
    assertEquals(expectedStorageFilePath, storageFile.getFilePath());
    assertEquals(expectedFileSizeBytes, storageFile.getFileSizeBytes());
    assertEquals(null, storageFile.getRowCount());
}

Example 39 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getTestStorageFiles.

/**
 * Builds and returns a list of test storage file object instances.
 *
 * @param s3KeyPrefix the S3 key prefix
 * @param relativeFilePaths the list of relative file paths that might include sub-directories,
 * @param setRowCount specifies if some storage files should get row count attribute set to a hard coded test value
 *
 * @return the newly created list of storage files
 */
public List<StorageFile> getTestStorageFiles(String s3KeyPrefix, List<String> relativeFilePaths, boolean setRowCount) {
    // Build a list of storage files.
    List<StorageFile> storageFiles = new ArrayList<>();
    for (String file : relativeFilePaths) {
        StorageFile storageFile = new StorageFile();
        storageFiles.add(storageFile);
        storageFile.setFilePath(s3KeyPrefix + "/" + file.replaceAll("\\\\", "/"));
        storageFile.setFileSizeBytes(AbstractServiceTest.FILE_SIZE_1_KB);
        if (setRowCount) {
            // Row count is an optional field, so let's not set it for one of the storage files - this is required for code coverage.
            storageFile.setRowCount(file.equals(AbstractServiceTest.LOCAL_FILES.get(0)) ? null : AbstractServiceTest.ROW_COUNT_1000);
        }
    }
    return storageFiles;
}

Example 40 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getStorageFiles.

/**
 * Creates and returns a list of test storage files.
 *
 * @return the list of storage files
 */
private List<StorageFile> getStorageFiles() {
    List<StorageFile> storageFiles = new ArrayList<>();
    StorageFile storageFile1 = new StorageFile();
    storageFiles.add(storageFile1);
    storageFile1.setFilePath("Folder/file1.gz");
    storageFile1.setFileSizeBytes(0L);
    storageFile1.setRowCount(0L);
    StorageFile storageFile2 = new StorageFile();
    storageFiles.add(storageFile2);
    storageFile2.setFilePath("Folder/file2.gz");
    storageFile2.setFileSizeBytes(2999L);
    storageFile2.setRowCount(1000L);
    StorageFile storageFile3 = new StorageFile();
    storageFiles.add(storageFile3);
    storageFile3.setFilePath("Folder/file3.gz");
    storageFile3.setFileSizeBytes(Long.MAX_VALUE);
    storageFile3.setRowCount(Long.MAX_VALUE);
    return storageFiles;
}