Examples with StorageUnit org.finra.herd.model.api.xml.StorageUnit used on opensource projects

Example 11 with StorageUnit

use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.

the class RelationalTableRegistrationServiceTest method testCreateRelationalTableRegistration.

@Test
public void testCreateRelationalTableRegistration() {
    // Create database entities required for relational table registration testing.
    relationalTableRegistrationServiceTestHelper.createDatabaseEntitiesForRelationalTableRegistrationTesting(BDEF_NAMESPACE, DATA_PROVIDER_NAME, STORAGE_NAME);
    // Pick one of the in-memory database tables to be registered as a relational table.
    String relationalSchemaName = "PUBLIC";
    String relationalTableName = BusinessObjectDefinitionEntity.TABLE_NAME.toUpperCase();
    // Create a relational table registration create request for a table that is part of the in-memory database setup as part of DAO mocks.
    RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest = new RelationalTableRegistrationCreateRequest(BDEF_NAMESPACE, BDEF_NAME, BDEF_DISPLAY_NAME, FORMAT_USAGE_CODE, DATA_PROVIDER_NAME, relationalSchemaName, relationalTableName, STORAGE_NAME);
    // Create a relational table registration.
    BusinessObjectData businessObjectData = relationalTableRegistrationService.createRelationalTableRegistration(relationalTableRegistrationCreateRequest, APPEND_TO_EXISTING_BUSINESS_OBJECT_DEFINTION_FALSE);
    // Create an expected storage unit.
    StorageUnit expectedStorageUnit = new StorageUnit();
    expectedStorageUnit.setStorage(new Storage(STORAGE_NAME, StoragePlatformEntity.RELATIONAL, relationalTableRegistrationServiceTestHelper.getStorageAttributes()));
    expectedStorageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
    // Create an expected business object data.
    BusinessObjectData expectedBusinessObjectData = new BusinessObjectData();
    expectedBusinessObjectData.setId(businessObjectData.getId());
    expectedBusinessObjectData.setNamespace(BDEF_NAMESPACE);
    expectedBusinessObjectData.setBusinessObjectDefinitionName(BDEF_NAME);
    expectedBusinessObjectData.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    expectedBusinessObjectData.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
    expectedBusinessObjectData.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
    expectedBusinessObjectData.setPartitionValue(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_VALUE);
    expectedBusinessObjectData.setSubPartitionValues(new ArrayList<>());
    expectedBusinessObjectData.setVersion(INITIAL_DATA_VERSION);
    expectedBusinessObjectData.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
    expectedBusinessObjectData.setLatestVersion(LATEST_VERSION_FLAG_SET);
    expectedBusinessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
    expectedBusinessObjectData.setStorageUnits(Collections.singletonList(expectedStorageUnit));
    expectedBusinessObjectData.setAttributes(new ArrayList<>());
    expectedBusinessObjectData.setBusinessObjectDataParents(new ArrayList<>());
    expectedBusinessObjectData.setBusinessObjectDataChildren(new ArrayList<>());
    // Validate the response.
    assertEquals(expectedBusinessObjectData, businessObjectData);
    // Create a business object format key.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE, INITIAL_FORMAT_VERSION);
    // Retrieve business object format that was created as part of the relational table registration.
    BusinessObjectFormat businessObjectFormat = businessObjectFormatService.getBusinessObjectFormat(businessObjectFormatKey);
    // Create an expected schema.
    Schema expectedSchema = new Schema();
    expectedSchema.setColumns(relationalTableRegistrationServiceTestHelper.getExpectedSchemaColumns());
    expectedSchema.setNullValue(EMPTY_STRING);
    // Build an expected business object format.
    BusinessObjectFormat expectedBusinessObjectFormat = new BusinessObjectFormat();
    expectedBusinessObjectFormat.setId(businessObjectFormat.getId());
    expectedBusinessObjectFormat.setNamespace(BDEF_NAMESPACE);
    expectedBusinessObjectFormat.setBusinessObjectDefinitionName(BDEF_NAME);
    expectedBusinessObjectFormat.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    expectedBusinessObjectFormat.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
    expectedBusinessObjectFormat.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
    expectedBusinessObjectFormat.setLatestVersion(LATEST_VERSION_FLAG_SET);
    expectedBusinessObjectFormat.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
    expectedBusinessObjectFormat.setBusinessObjectFormatParents(new ArrayList<>());
    expectedBusinessObjectFormat.setBusinessObjectFormatChildren(new ArrayList<>());
    expectedBusinessObjectFormat.setAttributeDefinitions(new ArrayList<>());
    expectedBusinessObjectFormat.setAttributes(Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_SCHEMA_NAME), relationalSchemaName), new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_TABLE_NAME), relationalTableName)));
    expectedBusinessObjectFormat.setSchema(expectedSchema);
    // Validate the newly created business object format.
    assertEquals(expectedBusinessObjectFormat, businessObjectFormat);
    // Create a business object definition key.
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(BDEF_NAMESPACE, BDEF_NAME);
    // Retrieve business object definition that was created as part of the relational table registration.
    BusinessObjectDefinition businessObjectDefinition = businessObjectDefinitionService.getBusinessObjectDefinition(businessObjectDefinitionKey, true);
    // Create an expected business object definition.
    BusinessObjectDefinition expectedBusinessObjectDefinition = new BusinessObjectDefinition();
    expectedBusinessObjectDefinition.setId(businessObjectDefinition.getId());
    expectedBusinessObjectDefinition.setNamespace(BDEF_NAMESPACE);
    expectedBusinessObjectDefinition.setBusinessObjectDefinitionName(BDEF_NAME);
    expectedBusinessObjectDefinition.setDataProviderName(DATA_PROVIDER_NAME);
    expectedBusinessObjectDefinition.setDisplayName(BDEF_DISPLAY_NAME);
    expectedBusinessObjectDefinition.setAttributes(new ArrayList<>());
    expectedBusinessObjectDefinition.setSampleDataFiles(new ArrayList<>());
    expectedBusinessObjectDefinition.setCreatedByUserId(businessObjectDefinition.getCreatedByUserId());
    expectedBusinessObjectDefinition.setLastUpdatedByUserId(businessObjectDefinition.getLastUpdatedByUserId());
    expectedBusinessObjectDefinition.setLastUpdatedOn(businessObjectDefinition.getLastUpdatedOn());
    expectedBusinessObjectDefinition.setBusinessObjectDefinitionChangeEvents(businessObjectDefinition.getBusinessObjectDefinitionChangeEvents());
    // Validate the newly created business object definition.
    assertEquals(expectedBusinessObjectDefinition, businessObjectDefinition);
}

Example 12 with StorageUnit

use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.

public void validateBusinessObjectData(String expectedNamespaceCode, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, String expectedBusinessObjectDataStatus, List<Attribute> expectedAttributes, String expectedStorageName, String expectedFileName, Long expectedFileSizeBytes, BusinessObjectData businessObjectData) {
    assertNotNull(businessObjectData);
    // Validate business object data alternate key values.
    assertEquals(expectedNamespaceCode, businessObjectData.getNamespace());
    assertEquals(expectedBusinessObjectDefinitionName, businessObjectData.getBusinessObjectDefinitionName());
    assertEquals(expectedBusinessObjectFormatUsage, businessObjectData.getBusinessObjectFormatUsage());
    assertEquals(expectedBusinessObjectFormatFileType, businessObjectData.getBusinessObjectFormatFileType());
    assertEquals(expectedBusinessObjectFormatVersion, Integer.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    // The business object data partition value must contain an UUID value.
    assertNotNull(businessObjectData.getPartitionValue());
    assertEquals(AbstractServiceTest.EXPECTED_UUID_SIZE, businessObjectData.getPartitionValue().length());
    assertEquals(AbstractServiceTest.NO_SUBPARTITION_VALUES, businessObjectData.getSubPartitionValues());
    assertEquals(AbstractServiceTest.INITIAL_DATA_VERSION, Integer.valueOf(businessObjectData.getVersion()));
    // Validate business object data status.
    assertTrue(businessObjectData.isLatestVersion());
    assertEquals(expectedBusinessObjectDataStatus, businessObjectData.getStatus());
    // Validate business object data attributes.
    businessObjectDefinitionServiceTestHelper.validateAttributes(expectedAttributes, businessObjectData.getAttributes());
    // Validate storage unit contents.
    assertEquals(1, businessObjectData.getStorageUnits().size());
    StorageUnit storageUnit = businessObjectData.getStorageUnits().get(0);
    assertEquals(expectedStorageName, storageUnit.getStorage().getName());
    String expectedStorageDirectoryPath = String.format("%s/%s/%s", AbstractServiceTest.ENVIRONMENT_NAME.trim().toLowerCase().replace('_', '-'), expectedNamespaceCode.trim().toLowerCase().replace('_', '-'), businessObjectData.getPartitionValue());
    assertEquals(expectedStorageDirectoryPath, storageUnit.getStorageDirectory().getDirectoryPath());
    assertEquals(1, storageUnit.getStorageFiles().size());
    StorageFile storageFile = storageUnit.getStorageFiles().get(0);
    String expectedStorageFilePath = String.format("%s/%s", expectedStorageDirectoryPath, expectedFileName);
    assertEquals(expectedStorageFilePath, storageFile.getFilePath());
    assertEquals(expectedFileSizeBytes, storageFile.getFileSizeBytes());
    assertEquals(null, storageFile.getRowCount());
}

Example 13 with StorageUnit

use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.

/**
 * Validates business object data against specified arguments and expected (hard coded) test values.
 *
 * @param expectedBusinessObjectDataId the expected business object data ID
 * @param expectedNamespace the expected namespace
 * @param expectedBusinessObjectDefinitionName the expected business object definition name
 * @param expectedBusinessObjectFormatUsage the expected business object format usage
 * @param expectedBusinessObjectFormatFileType the expected business object format file type
 * @param expectedBusinessObjectFormatVersion the expected business object format version
 * @param expectedBusinessObjectDataPartitionValue the expected partition value for this business object data
 * @param expectedBusinessObjectDataVersion the expected business object data version
 * @param expectedLatestVersion the expected business
 * @param expectedStatusCode the expected business object data status code
 * @param expectedStorageName the expected storage name
 * @param expectedStorageDirectoryPath the expected storage directory path
 * @param expectedStorageFiles the expected storage files
 * @param expectedAttributes the expected attributes
 * @param actualBusinessObjectData the business object data availability object instance to be validated
 */
public void validateBusinessObjectData(Integer expectedBusinessObjectDataId, String expectedNamespace, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, String expectedBusinessObjectDataPartitionValue, List<String> expectedBusinessObjectDataSubPartitionValues, Integer expectedBusinessObjectDataVersion, Boolean expectedLatestVersion, String expectedStatusCode, String expectedStorageName, String expectedStorageDirectoryPath, List<StorageFile> expectedStorageFiles, List<Attribute> expectedAttributes, BusinessObjectData actualBusinessObjectData) {
    validateBusinessObjectData(expectedBusinessObjectDataId, expectedNamespace, expectedBusinessObjectDefinitionName, expectedBusinessObjectFormatUsage, expectedBusinessObjectFormatFileType, expectedBusinessObjectFormatVersion, expectedBusinessObjectDataPartitionValue, expectedBusinessObjectDataSubPartitionValues, expectedBusinessObjectDataVersion, expectedLatestVersion, expectedStatusCode, actualBusinessObjectData);
    // We expected test business object data to contain a single storage unit.
    assertEquals(1, actualBusinessObjectData.getStorageUnits().size());
    StorageUnit actualStorageUnit = actualBusinessObjectData.getStorageUnits().get(0);
    assertEquals(expectedStorageName, actualStorageUnit.getStorage().getName());
    assertEquals(expectedStorageDirectoryPath, actualStorageUnit.getStorageDirectory() != null ? actualStorageUnit.getStorageDirectory().getDirectoryPath() : null);
    AbstractServiceTest.assertEqualsIgnoreOrder("storage files", expectedStorageFiles, actualStorageUnit.getStorageFiles());
    assertEquals(expectedAttributes, actualBusinessObjectData.getAttributes());
}

Example 14 with StorageUnit

use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.

the class BusinessObjectDataInvalidateUnregisteredHelperTest method testInvalidateUnregisteredBusinessObjectDataS31Herd0.

/**
 * Test case where S3 has 1 object, and herd has no object registered. Expects one new registration in INVALID status.
 */
@Test
public void testInvalidateUnregisteredBusinessObjectDataS31Herd0() throws Exception {
    BusinessObjectDataInvalidateUnregisteredRequest request = new BusinessObjectDataInvalidateUnregisteredRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, StorageEntity.MANAGED_STORAGE);
    // Given a business object format
    // Given an object in S3
    BusinessObjectFormatEntity businessObjectFormatEntity;
    try {
        businessObjectFormatEntity = businessObjectFormatServiceTestHelper.createBusinessObjectFormat(request);
        businessObjectDataServiceTestHelper.createS3Object(businessObjectFormatEntity, request, 0);
    } catch (Exception e) {
        throw new IllegalArgumentException("Test failed during setup. Most likely setup or developer error.", e);
    }
    // Call API
    BusinessObjectDataInvalidateUnregisteredResponse actualResponse = businessObjectDataInvalidateUnregisteredHelper.invalidateUnregisteredBusinessObjectData(request);
    // Make assertions
    Assert.assertNotNull("response business object datas is null", actualResponse.getRegisteredBusinessObjectDataList());
    Assert.assertEquals("response business object datas size", 1, actualResponse.getRegisteredBusinessObjectDataList().size());
    {
        BusinessObjectData businessObjectData = actualResponse.getRegisteredBusinessObjectDataList().get(0);
        Assert.assertEquals("response business object data[0] version", 0, businessObjectData.getVersion());
        Assert.assertEquals("response business object data[0] status", BusinessObjectDataInvalidateUnregisteredHelper.UNREGISTERED_STATUS, businessObjectData.getStatus());
        Assert.assertNotNull("response business object data[0] storage units is null", businessObjectData.getStorageUnits());
        Assert.assertEquals("response business object data[0] storage units size", 1, businessObjectData.getStorageUnits().size());
        {
            String expectedS3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.createBusinessObjectDataKey(businessObjectData), STORAGE_NAME);
            StorageUnit storageUnit = businessObjectData.getStorageUnits().get(0);
            Assert.assertNotNull("response business object data[0] storage unit[0] storage directory is null", storageUnit.getStorageDirectory());
            Assert.assertEquals("response business object data[0] storage unit[0] storage directory path", expectedS3KeyPrefix, storageUnit.getStorageDirectory().getDirectoryPath());
        }
    }
}

Example 15 with StorageUnit

use of org.finra.herd.model.api.xml.StorageUnit in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateDownloadedS3FilesFileCountMismatch.

@Test
public void testValidateDownloadedS3FilesFileCountMismatch() throws IOException {
    File targetLocalDirectory = Paths.get(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX).toFile();
    createLocalFiles(targetLocalDirectory.getPath(), FILE_SIZE_1_KB);
    createLocalFile(targetLocalDirectory.getPath(), "EXTRA_FILE", FILE_SIZE_1_KB);
    StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB);
    // Try to validate the local files, when number of files does not match to the storage unit information.
    try {
        storageFileHelper.validateDownloadedS3Files(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX, storageUnit);
        fail("Should throw a RuntimeException when number of local files does not match to the storage unit information.");
    } catch (RuntimeException e) {
        String expectedErrMsg = String.format("Number of downloaded files does not match the storage unit information (expected %d files, actual %d files).", storageUnit.getStorageFiles().size(), LOCAL_FILES.size() + 1);
        assertEquals(expectedErrMsg, e.getMessage());
    }
}