Examples with BusinessObjectFormatEntity org.finra.herd.model.jpa.BusinessObjectFormatEntity used on opensource projects

Example 36 with BusinessObjectFormatEntity

use of org.finra.herd.model.jpa.BusinessObjectFormatEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testDeleteBusinessObjectFormatNotLatestVersion.

@Test
public void testDeleteBusinessObjectFormatNotLatestVersion() {
    // Create and persist a business object format which is not marked as the latest version.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, false, PARTITION_KEY);
    // Delete the business object format.
    BusinessObjectFormat deletedBusinessObjectFormat = businessObjectFormatService.deleteBusinessObjectFormat(new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION));
    // Validate the returned object.
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormat(businessObjectFormatEntity.getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, false, PARTITION_KEY, FORMAT_DESCRIPTION, NO_ATTRIBUTES, NO_ATTRIBUTE_DEFINITIONS, NO_SCHEMA, deletedBusinessObjectFormat);
    // Ensure that this business object format is no longer there.
    assertNull(businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION)));
}

Example 37 with BusinessObjectFormatEntity

use of org.finra.herd.model.jpa.BusinessObjectFormatEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testUpdateBusinessObjectFormatMissingOptionalParameters.

@Test
public void testUpdateBusinessObjectFormatMissingOptionalParameters() {
    // Create and persist a business object definition.
    businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION);
    // Create and persist a valid business object format without description and with schema without any partitioning columns.
    BusinessObjectFormatEntity originalBusinessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_COMMA, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), null);
    // Create a new partition key group for the update request.
    partitionKeyGroupDaoTestHelper.createPartitionKeyGroupEntity(PARTITION_KEY_GROUP_2);
    // Perform an update by changing the description to a blank text, replacing attributes with
    // an attribute with a blank text value, and replacing schema with a new schema not having partition columns.
    Schema testSchema2 = businessObjectFormatServiceTestHelper.getTestSchema2();
    testSchema2.setPartitions(null);
    for (String blankText : Arrays.asList(BLANK_TEXT, EMPTY_STRING, null)) {
        BusinessObjectFormatUpdateRequest request = businessObjectFormatServiceTestHelper.createBusinessObjectFormatUpdateRequest(blankText, Arrays.asList(new Attribute(ATTRIBUTE_NAME_4_MIXED_CASE, blankText)), testSchema2);
        BusinessObjectFormat resultBusinessObjectFormat = businessObjectFormatService.updateBusinessObjectFormat(new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION), request);
        // Validate the returned object.
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormat(originalBusinessObjectFormatEntity.getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, blankText, Arrays.asList(new Attribute(ATTRIBUTE_NAME_4_MIXED_CASE, blankText)), NO_ATTRIBUTE_DEFINITIONS, testSchema2, resultBusinessObjectFormat);
    }
}

Example 38 with BusinessObjectFormatEntity

use of org.finra.herd.model.jpa.BusinessObjectFormatEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testCreateBusinessObjectFormatCreateSecondVersionNoPartitioningColumns.

@Test
public void testCreateBusinessObjectFormatCreateSecondVersionNoPartitioningColumns() {
    // Create and persist an initial version of a business object format with schema without any partitioning columns.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(RANDOM_SUFFIX), null);
    // Create a second version of the business object format without partitioning columns.
    Schema testSchema = businessObjectFormatServiceTestHelper.getTestSchema();
    testSchema.setPartitions(null);
    BusinessObjectFormatCreateRequest request = businessObjectFormatServiceTestHelper.createBusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, businessObjectDefinitionServiceTestHelper.getNewAttributes(), businessObjectFormatServiceTestHelper.getTestAttributeDefinitions(), testSchema);
    BusinessObjectFormat resultBusinessObjectFormat = businessObjectFormatService.createBusinessObjectFormat(request);
    // Validate the returned object.
    Schema expectedSchema = businessObjectFormatServiceTestHelper.getTestSchema();
    expectedSchema.setPartitions(null);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormat(null, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, FORMAT_DESCRIPTION, businessObjectDefinitionServiceTestHelper.getNewAttributes(), businessObjectFormatServiceTestHelper.getTestAttributeDefinitions(), expectedSchema, resultBusinessObjectFormat);
    // Check if we now have only one latest format version - our first version is not marked as latest anymore.
    // Please note that we do not have to validate if the first format version is not marked as "latest" now, since having more that one
    // format versions with the latestVersion flag set to Yes produces exception in herdDao.getBusinessObjectFormatByAltKey() method.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, null));
    assertEquals(SECOND_FORMAT_VERSION, businessObjectFormatEntity.getBusinessObjectFormatVersion());
    assertEquals(true, businessObjectFormatEntity.getLatestVersion());
}

Example 39 with BusinessObjectFormatEntity

use of org.finra.herd.model.jpa.BusinessObjectFormatEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testCreateBusinessObjectFormatWithParents.

@Test
public void testCreateBusinessObjectFormatWithParents() {
    // Create relative database entities including a business object definition.
    businessObjectFormatServiceTestHelper.createTestDatabaseEntitiesForBusinessObjectFormatTesting(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_FILE_TYPE_CODE, PARTITION_KEY_GROUP);
    businessObjectFormatServiceTestHelper.createTestDatabaseEntitiesForBusinessObjectFormatTesting(NAMESPACE_2, DATA_PROVIDER_NAME_2, BDEF_NAME_2, FORMAT_FILE_TYPE_CODE_2, PARTITION_KEY_GROUP_2);
    // parent business object format
    BusinessObjectFormatCreateRequest request = businessObjectFormatServiceTestHelper.createBusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, NO_FORMAT_DESCRIPTION, businessObjectDefinitionServiceTestHelper.getNewAttributes(), NO_ATTRIBUTE_DEFINITIONS, NO_SCHEMA);
    List<BusinessObjectFormatKey> businessObjectFormatParents = Arrays.asList(new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, null));
    BusinessObjectFormatCreateRequest request2 = businessObjectFormatServiceTestHelper.createBusinessObjectFormatCreateRequest(NAMESPACE_2, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, PARTITION_KEY, NO_FORMAT_DESCRIPTION, businessObjectDefinitionServiceTestHelper.getNewAttributes(), NO_ATTRIBUTE_DEFINITIONS, NO_SCHEMA);
    BusinessObjectFormat resultBusinessObjectFormatV0 = businessObjectFormatService.createBusinessObjectFormat(request);
    BusinessObjectFormat resultBusinessObjectFormatChildV0 = businessObjectFormatService.createBusinessObjectFormat(request2);
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, null);
    BusinessObjectFormatKey childBusinessObjectFormatKey = new BusinessObjectFormatKey(NAMESPACE_2, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, null);
    // create parent, child relationship
    BusinessObjectFormatEntity formatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(businessObjectFormatKey);
    BusinessObjectFormatEntity childFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(childBusinessObjectFormatKey);
    childFormatEntity.getBusinessObjectFormatParents().add(formatEntity);
    formatEntity.getBusinessObjectFormatChildren().add(childFormatEntity);
    // create a new version of the child format
    BusinessObjectFormat resultBusinessObjectFormatChildV1 = businessObjectFormatService.createBusinessObjectFormat(request2);
    BusinessObjectFormat expectedBusinessObjectFormatChild = resultBusinessObjectFormatChildV0;
    expectedBusinessObjectFormatChild.setId(resultBusinessObjectFormatChildV1.getId());
    expectedBusinessObjectFormatChild.setBusinessObjectFormatVersion(resultBusinessObjectFormatChildV1.getBusinessObjectFormatVersion());
    expectedBusinessObjectFormatChild.setBusinessObjectFormatParents(resultBusinessObjectFormatChildV1.getBusinessObjectFormatParents());
    assertEquals(resultBusinessObjectFormatChildV1.getBusinessObjectFormatParents().size(), 1);
    assertEquals(expectedBusinessObjectFormatChild, resultBusinessObjectFormatChildV1);
    // create a new version of the parent
    BusinessObjectFormat resultBusinessObjectFormatParentV2 = businessObjectFormatService.createBusinessObjectFormat(request);
    assertEquals(resultBusinessObjectFormatParentV2.getBusinessObjectFormatChildren().size(), 1);
    BusinessObjectFormat expectedBusinessObjectParent = resultBusinessObjectFormatV0;
    expectedBusinessObjectParent.setId(resultBusinessObjectFormatParentV2.getId());
    expectedBusinessObjectParent.setBusinessObjectFormatVersion(resultBusinessObjectFormatParentV2.getBusinessObjectFormatVersion());
    expectedBusinessObjectParent.setBusinessObjectFormatChildren(resultBusinessObjectFormatParentV2.getBusinessObjectFormatChildren());
    assertEquals(expectedBusinessObjectParent, resultBusinessObjectFormatParentV2);
}

Example 40 with BusinessObjectFormatEntity

use of org.finra.herd.model.jpa.BusinessObjectFormatEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
    // Get storage entity.
    StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    // Expected business object format file type to Hive file format mapping.
    HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
    businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
        // Prepare test data for the respective business object format file type.
        List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
        String partitionKey = partitionColumns.get(0).getName();
        BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
        for (String partitionValue : UNSORTED_PARTITION_VALUES) {
            BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
            String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
            // Create two storage files.
            for (int i = 0; i < 2; i++) {
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
            herdDao.saveAndRefresh(storageUnitEntity);
            herdDao.saveAndRefresh(businessObjectDataEntity);
        }
        // Retrieve business object format ddl.
        BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
        request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
        BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        // Validate the results.
        String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
        String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
    }
}