use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.
the class RelationalTableRegistrationServiceTest method testCreateRelationalTableRegistration.
@Test
public void testCreateRelationalTableRegistration() {
// Create database entities required for relational table registration testing.
relationalTableRegistrationServiceTestHelper.createDatabaseEntitiesForRelationalTableRegistrationTesting(BDEF_NAMESPACE, DATA_PROVIDER_NAME, STORAGE_NAME);
// Pick one of the in-memory database tables to be registered as a relational table.
String relationalSchemaName = "PUBLIC";
String relationalTableName = BusinessObjectDefinitionEntity.TABLE_NAME.toUpperCase();
// Create a relational table registration create request for a table that is part of the in-memory database setup as part of DAO mocks.
RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest = new RelationalTableRegistrationCreateRequest(BDEF_NAMESPACE, BDEF_NAME, BDEF_DISPLAY_NAME, FORMAT_USAGE_CODE, DATA_PROVIDER_NAME, relationalSchemaName, relationalTableName, STORAGE_NAME);
// Create a relational table registration.
BusinessObjectData businessObjectData = relationalTableRegistrationService.createRelationalTableRegistration(relationalTableRegistrationCreateRequest, APPEND_TO_EXISTING_BUSINESS_OBJECT_DEFINTION_FALSE);
// Create an expected storage unit.
StorageUnit expectedStorageUnit = new StorageUnit();
expectedStorageUnit.setStorage(new Storage(STORAGE_NAME, StoragePlatformEntity.RELATIONAL, relationalTableRegistrationServiceTestHelper.getStorageAttributes()));
expectedStorageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
// Create an expected business object data.
BusinessObjectData expectedBusinessObjectData = new BusinessObjectData();
expectedBusinessObjectData.setId(businessObjectData.getId());
expectedBusinessObjectData.setNamespace(BDEF_NAMESPACE);
expectedBusinessObjectData.setBusinessObjectDefinitionName(BDEF_NAME);
expectedBusinessObjectData.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
expectedBusinessObjectData.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
expectedBusinessObjectData.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
expectedBusinessObjectData.setPartitionValue(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_VALUE);
expectedBusinessObjectData.setSubPartitionValues(new ArrayList<>());
expectedBusinessObjectData.setVersion(INITIAL_DATA_VERSION);
expectedBusinessObjectData.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
expectedBusinessObjectData.setLatestVersion(LATEST_VERSION_FLAG_SET);
expectedBusinessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
expectedBusinessObjectData.setStorageUnits(Collections.singletonList(expectedStorageUnit));
expectedBusinessObjectData.setAttributes(new ArrayList<>());
expectedBusinessObjectData.setBusinessObjectDataParents(new ArrayList<>());
expectedBusinessObjectData.setBusinessObjectDataChildren(new ArrayList<>());
// Validate the response.
assertEquals(expectedBusinessObjectData, businessObjectData);
// Create a business object format key.
BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE, INITIAL_FORMAT_VERSION);
// Retrieve business object format that was created as part of the relational table registration.
BusinessObjectFormat businessObjectFormat = businessObjectFormatService.getBusinessObjectFormat(businessObjectFormatKey);
// Create an expected schema.
Schema expectedSchema = new Schema();
expectedSchema.setColumns(relationalTableRegistrationServiceTestHelper.getExpectedSchemaColumns());
expectedSchema.setNullValue(EMPTY_STRING);
// Build an expected business object format.
BusinessObjectFormat expectedBusinessObjectFormat = new BusinessObjectFormat();
expectedBusinessObjectFormat.setId(businessObjectFormat.getId());
expectedBusinessObjectFormat.setNamespace(BDEF_NAMESPACE);
expectedBusinessObjectFormat.setBusinessObjectDefinitionName(BDEF_NAME);
expectedBusinessObjectFormat.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
expectedBusinessObjectFormat.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
expectedBusinessObjectFormat.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
expectedBusinessObjectFormat.setLatestVersion(LATEST_VERSION_FLAG_SET);
expectedBusinessObjectFormat.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
expectedBusinessObjectFormat.setBusinessObjectFormatParents(new ArrayList<>());
expectedBusinessObjectFormat.setBusinessObjectFormatChildren(new ArrayList<>());
expectedBusinessObjectFormat.setAttributeDefinitions(new ArrayList<>());
expectedBusinessObjectFormat.setAttributes(Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_SCHEMA_NAME), relationalSchemaName), new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_TABLE_NAME), relationalTableName)));
expectedBusinessObjectFormat.setSchema(expectedSchema);
// Validate the newly created business object format.
assertEquals(expectedBusinessObjectFormat, businessObjectFormat);
// Create a business object definition key.
BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(BDEF_NAMESPACE, BDEF_NAME);
// Retrieve business object definition that was created as part of the relational table registration.
BusinessObjectDefinition businessObjectDefinition = businessObjectDefinitionService.getBusinessObjectDefinition(businessObjectDefinitionKey, true);
// Create an expected business object definition.
BusinessObjectDefinition expectedBusinessObjectDefinition = new BusinessObjectDefinition();
expectedBusinessObjectDefinition.setId(businessObjectDefinition.getId());
expectedBusinessObjectDefinition.setNamespace(BDEF_NAMESPACE);
expectedBusinessObjectDefinition.setBusinessObjectDefinitionName(BDEF_NAME);
expectedBusinessObjectDefinition.setDataProviderName(DATA_PROVIDER_NAME);
expectedBusinessObjectDefinition.setDisplayName(BDEF_DISPLAY_NAME);
expectedBusinessObjectDefinition.setAttributes(new ArrayList<>());
expectedBusinessObjectDefinition.setSampleDataFiles(new ArrayList<>());
expectedBusinessObjectDefinition.setCreatedByUserId(businessObjectDefinition.getCreatedByUserId());
expectedBusinessObjectDefinition.setLastUpdatedByUserId(businessObjectDefinition.getLastUpdatedByUserId());
expectedBusinessObjectDefinition.setLastUpdatedOn(businessObjectDefinition.getLastUpdatedOn());
expectedBusinessObjectDefinition.setBusinessObjectDefinitionChangeEvents(businessObjectDefinition.getBusinessObjectDefinitionChangeEvents());
// Validate the newly created business object definition.
assertEquals(expectedBusinessObjectDefinition, businessObjectDefinition);
}
use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.
the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.
public void validateBusinessObjectData(String expectedNamespaceCode, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, String expectedBusinessObjectDataStatus, List<Attribute> expectedAttributes, String expectedStorageName, String expectedFileName, Long expectedFileSizeBytes, BusinessObjectData businessObjectData) {
assertNotNull(businessObjectData);
// Validate business object data alternate key values.
assertEquals(expectedNamespaceCode, businessObjectData.getNamespace());
assertEquals(expectedBusinessObjectDefinitionName, businessObjectData.getBusinessObjectDefinitionName());
assertEquals(expectedBusinessObjectFormatUsage, businessObjectData.getBusinessObjectFormatUsage());
assertEquals(expectedBusinessObjectFormatFileType, businessObjectData.getBusinessObjectFormatFileType());
assertEquals(expectedBusinessObjectFormatVersion, Integer.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
// The business object data partition value must contain an UUID value.
assertNotNull(businessObjectData.getPartitionValue());
assertEquals(AbstractServiceTest.EXPECTED_UUID_SIZE, businessObjectData.getPartitionValue().length());
assertEquals(AbstractServiceTest.NO_SUBPARTITION_VALUES, businessObjectData.getSubPartitionValues());
assertEquals(AbstractServiceTest.INITIAL_DATA_VERSION, Integer.valueOf(businessObjectData.getVersion()));
// Validate business object data status.
assertTrue(businessObjectData.isLatestVersion());
assertEquals(expectedBusinessObjectDataStatus, businessObjectData.getStatus());
// Validate business object data attributes.
businessObjectDefinitionServiceTestHelper.validateAttributes(expectedAttributes, businessObjectData.getAttributes());
// Validate storage unit contents.
assertEquals(1, businessObjectData.getStorageUnits().size());
StorageUnit storageUnit = businessObjectData.getStorageUnits().get(0);
assertEquals(expectedStorageName, storageUnit.getStorage().getName());
String expectedStorageDirectoryPath = String.format("%s/%s/%s", AbstractServiceTest.ENVIRONMENT_NAME.trim().toLowerCase().replace('_', '-'), expectedNamespaceCode.trim().toLowerCase().replace('_', '-'), businessObjectData.getPartitionValue());
assertEquals(expectedStorageDirectoryPath, storageUnit.getStorageDirectory().getDirectoryPath());
assertEquals(1, storageUnit.getStorageFiles().size());
StorageFile storageFile = storageUnit.getStorageFiles().get(0);
String expectedStorageFilePath = String.format("%s/%s", expectedStorageDirectoryPath, expectedFileName);
assertEquals(expectedStorageFilePath, storageFile.getFilePath());
assertEquals(expectedFileSizeBytes, storageFile.getFileSizeBytes());
assertEquals(null, storageFile.getRowCount());
}
use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.
the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.
/**
* Validates business object data against specified arguments and expected (hard coded) test values.
*
* @param expectedBusinessObjectDataId the expected business object data ID
* @param expectedNamespace the expected namespace
* @param expectedBusinessObjectDefinitionName the expected business object definition name
* @param expectedBusinessObjectFormatUsage the expected business object format usage
* @param expectedBusinessObjectFormatFileType the expected business object format file type
* @param expectedBusinessObjectFormatVersion the expected business object format version
* @param expectedBusinessObjectDataPartitionValue the expected partition value for this business object data
* @param expectedBusinessObjectDataVersion the expected business object data version
* @param expectedLatestVersion the expected business
* @param expectedStatusCode the expected business object data status code
* @param expectedStorageName the expected storage name
* @param expectedStorageDirectoryPath the expected storage directory path
* @param expectedStorageFiles the expected storage files
* @param expectedAttributes the expected attributes
* @param actualBusinessObjectData the business object data availability object instance to be validated
*/
public void validateBusinessObjectData(Integer expectedBusinessObjectDataId, String expectedNamespace, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, String expectedBusinessObjectDataPartitionValue, List<String> expectedBusinessObjectDataSubPartitionValues, Integer expectedBusinessObjectDataVersion, Boolean expectedLatestVersion, String expectedStatusCode, String expectedStorageName, String expectedStorageDirectoryPath, List<StorageFile> expectedStorageFiles, List<Attribute> expectedAttributes, BusinessObjectData actualBusinessObjectData) {
validateBusinessObjectData(expectedBusinessObjectDataId, expectedNamespace, expectedBusinessObjectDefinitionName, expectedBusinessObjectFormatUsage, expectedBusinessObjectFormatFileType, expectedBusinessObjectFormatVersion, expectedBusinessObjectDataPartitionValue, expectedBusinessObjectDataSubPartitionValues, expectedBusinessObjectDataVersion, expectedLatestVersion, expectedStatusCode, actualBusinessObjectData);
// We expected test business object data to contain a single storage unit.
assertEquals(1, actualBusinessObjectData.getStorageUnits().size());
StorageUnit actualStorageUnit = actualBusinessObjectData.getStorageUnits().get(0);
assertEquals(expectedStorageName, actualStorageUnit.getStorage().getName());
assertEquals(expectedStorageDirectoryPath, actualStorageUnit.getStorageDirectory() != null ? actualStorageUnit.getStorageDirectory().getDirectoryPath() : null);
AbstractServiceTest.assertEqualsIgnoreOrder("storage files", expectedStorageFiles, actualStorageUnit.getStorageFiles());
assertEquals(expectedAttributes, actualBusinessObjectData.getAttributes());
}
use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.
the class BusinessObjectDataInvalidateUnregisteredHelperTest method testInvalidateUnregisteredBusinessObjectDataS31Herd0.
/**
* Test case where S3 has 1 object, and herd has no object registered. Expects one new registration in INVALID status.
*/
@Test
public void testInvalidateUnregisteredBusinessObjectDataS31Herd0() throws Exception {
BusinessObjectDataInvalidateUnregisteredRequest request = new BusinessObjectDataInvalidateUnregisteredRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, StorageEntity.MANAGED_STORAGE);
// Given a business object format
// Given an object in S3
BusinessObjectFormatEntity businessObjectFormatEntity;
try {
businessObjectFormatEntity = businessObjectFormatServiceTestHelper.createBusinessObjectFormat(request);
businessObjectDataServiceTestHelper.createS3Object(businessObjectFormatEntity, request, 0);
} catch (Exception e) {
throw new IllegalArgumentException("Test failed during setup. Most likely setup or developer error.", e);
}
// Call API
BusinessObjectDataInvalidateUnregisteredResponse actualResponse = businessObjectDataInvalidateUnregisteredHelper.invalidateUnregisteredBusinessObjectData(request);
// Make assertions
Assert.assertNotNull("response business object datas is null", actualResponse.getRegisteredBusinessObjectDataList());
Assert.assertEquals("response business object datas size", 1, actualResponse.getRegisteredBusinessObjectDataList().size());
{
BusinessObjectData businessObjectData = actualResponse.getRegisteredBusinessObjectDataList().get(0);
Assert.assertEquals("response business object data[0] version", 0, businessObjectData.getVersion());
Assert.assertEquals("response business object data[0] status", BusinessObjectDataInvalidateUnregisteredHelper.UNREGISTERED_STATUS, businessObjectData.getStatus());
Assert.assertNotNull("response business object data[0] storage units is null", businessObjectData.getStorageUnits());
Assert.assertEquals("response business object data[0] storage units size", 1, businessObjectData.getStorageUnits().size());
{
String expectedS3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.createBusinessObjectDataKey(businessObjectData), STORAGE_NAME);
StorageUnit storageUnit = businessObjectData.getStorageUnits().get(0);
Assert.assertNotNull("response business object data[0] storage unit[0] storage directory is null", storageUnit.getStorageDirectory());
Assert.assertEquals("response business object data[0] storage unit[0] storage directory path", expectedS3KeyPrefix, storageUnit.getStorageDirectory().getDirectoryPath());
}
}
}
use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.
the class StorageFileHelperTest method testValidateDownloadedS3FilesFileCountMismatch.
@Test
public void testValidateDownloadedS3FilesFileCountMismatch() throws IOException {
File targetLocalDirectory = Paths.get(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX).toFile();
createLocalFiles(targetLocalDirectory.getPath(), FILE_SIZE_1_KB);
createLocalFile(targetLocalDirectory.getPath(), "EXTRA_FILE", FILE_SIZE_1_KB);
StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB);
// Try to validate the local files, when number of files does not match to the storage unit information.
try {
storageFileHelper.validateDownloadedS3Files(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX, storageUnit);
fail("Should throw a RuntimeException when number of local files does not match to the storage unit information.");
} catch (RuntimeException e) {
String expectedErrMsg = String.format("Number of downloaded files does not match the storage unit information (expected %d files, actual %d files).", storageUnit.getStorageFiles().size(), LOCAL_FILES.size() + 1);
assertEquals(expectedErrMsg, e.getMessage());
}
}
Aggregations