Examples with BusinessObjectDataEntity org.finra.herd.model.jpa.BusinessObjectDataEntity used on opensource projects

Example 96 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method createData.

private void createData(String storageUnitDirectory, boolean s3Managed, Collection<String> files) {
    // Create and persist a "pre-registration" business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
    // Create and persist a business object data entity.
    BusinessObjectDataEntity bod = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, businessObjectDataStatusEntity.getCode());
    StorageEntity s;
    if (s3Managed) {
        s = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    } else {
        s = super.storageDaoTestHelper.createStorageEntity(STORAGE_NAME, STORAGE_PLATFORM_CODE);
    }
    StorageUnitEntity su = super.storageUnitDaoTestHelper.createStorageUnitEntity(s, bod, StorageUnitStatusEntity.ENABLED, storageUnitDirectory);
    for (String file : files) {
        StorageFileEntity f = super.storageFileDaoTestHelper.createStorageFileEntity(su, file, FILE_SIZE_1_KB, ROW_COUNT_1000);
        su.getStorageFiles().add(f);
    }
}

Example 97 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getNewBusinessObjectDataCreateRequest.

/**
 * Gets a newly created business object data create request.
 *
 * @param includeAttributes If true, attribute definitions and attributes will be included. Otherwise, not.
 *
 * @return the business object create request.
 */
public BusinessObjectDataCreateRequest getNewBusinessObjectDataCreateRequest(boolean includeAttributes) {
    // Crete a test business object format (and associated data).
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(includeAttributes);
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity();
    // Create a request to create business object data.
    BusinessObjectDataCreateRequest businessObjectDataCreateRequest = new BusinessObjectDataCreateRequest();
    businessObjectDataCreateRequest.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
    businessObjectDataCreateRequest.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
    businessObjectDataCreateRequest.setBusinessObjectFormatUsage(businessObjectFormatEntity.getUsage());
    businessObjectDataCreateRequest.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
    businessObjectDataCreateRequest.setBusinessObjectFormatVersion(businessObjectFormatEntity.getBusinessObjectFormatVersion());
    businessObjectDataCreateRequest.setPartitionKey(businessObjectFormatEntity.getPartitionKey());
    businessObjectDataCreateRequest.setPartitionValue(AbstractServiceTest.PARTITION_VALUE);
    businessObjectDataCreateRequest.setSubPartitionValues(AbstractServiceTest.SUBPARTITION_VALUES);
    List<StorageUnitCreateRequest> storageUnits = new ArrayList<>();
    businessObjectDataCreateRequest.setStorageUnits(storageUnits);
    StorageUnitCreateRequest storageUnit = new StorageUnitCreateRequest();
    storageUnits.add(storageUnit);
    storageUnit.setStorageName(storageEntity.getName());
    StorageDirectory storageDirectory = new StorageDirectory();
    storageUnit.setStorageDirectory(storageDirectory);
    storageDirectory.setDirectoryPath("Folder");
    List<StorageFile> storageFiles = new ArrayList<>();
    storageUnit.setStorageFiles(storageFiles);
    StorageFile storageFile1 = new StorageFile();
    storageFiles.add(storageFile1);
    storageFile1.setFilePath("Folder/file1.gz");
    storageFile1.setFileSizeBytes(0L);
    storageFile1.setRowCount(0L);
    StorageFile storageFile2 = new StorageFile();
    storageFiles.add(storageFile2);
    storageFile2.setFilePath("Folder/file2.gz");
    storageFile2.setFileSizeBytes(2999L);
    storageFile2.setRowCount(1000L);
    StorageFile storageFile3 = new StorageFile();
    storageFiles.add(storageFile3);
    storageFile3.setFilePath("Folder/file3.gz");
    storageFile3.setFileSizeBytes(Long.MAX_VALUE);
    storageFile3.setRowCount(Long.MAX_VALUE);
    if (includeAttributes) {
        businessObjectDataCreateRequest.setAttributes(businessObjectDefinitionServiceTestHelper.getNewAttributes());
    }
    List<BusinessObjectDataKey> businessObjectDataParents = new ArrayList<>();
    businessObjectDataCreateRequest.setBusinessObjectDataParents(businessObjectDataParents);
    // Create 2 parents.
    for (int i = 0; i < 2; i++) {
        BusinessObjectDataEntity parentBusinessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity();
        BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey();
        businessObjectDataKey.setNamespace(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getNamespace().getCode());
        businessObjectDataKey.setBusinessObjectDefinitionName(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getName());
        businessObjectDataKey.setBusinessObjectFormatUsage(parentBusinessObjectDataEntity.getBusinessObjectFormat().getUsage());
        businessObjectDataKey.setBusinessObjectFormatFileType(parentBusinessObjectDataEntity.getBusinessObjectFormat().getFileType().getCode());
        businessObjectDataKey.setBusinessObjectFormatVersion(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectFormatVersion());
        businessObjectDataKey.setPartitionValue(parentBusinessObjectDataEntity.getPartitionValue());
        businessObjectDataKey.setBusinessObjectDataVersion(parentBusinessObjectDataEntity.getVersion());
        businessObjectDataKey.setSubPartitionValues(businessObjectDataHelper.getSubPartitionValues(parentBusinessObjectDataEntity));
        businessObjectDataParents.add(businessObjectDataKey);
    }
    return businessObjectDataCreateRequest;
}

Example 98 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels.

/**
 * Creates and persists VALID two-level partitioned business object data with "available" storage units in an S3 storage.
 *
 * @param partitions the list of partitions, where each is represented by a primary value and a sub-partition value
 *
 * @return the list of created storage unit entities
 */
public List<StorageUnitEntity> createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(List<List<String>> partitions) {
    // Create a list of storage unit entities to be returned.
    List<StorageUnitEntity> result = new ArrayList<>();
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, "DATE", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME, "STRING", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.COLUMN_NAME, "NUMBER", AbstractServiceTest.COLUMN_SIZE, AbstractServiceTest.NO_COLUMN_REQUIRED, AbstractServiceTest.COLUMN_DEFAULT_VALUE, AbstractServiceTest.COLUMN_DESCRIPTION));
    // Use the first two columns as partition columns.
    List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 2);
    // Create a business object format entity with the schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FORMAT_DESCRIPTION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, AbstractServiceTest.NO_PARTITION_KEY_GROUP, AbstractServiceTest.NO_ATTRIBUTES, AbstractServiceTest.SCHEMA_DELIMITER_PIPE, AbstractServiceTest.SCHEMA_ESCAPE_CHARACTER_BACKSLASH, AbstractServiceTest.SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
    // Create an S3 storage entity.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    for (List<String> partition : partitions) {
        // Build an S3 key prefix according to the herd S3 naming convention.
        String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(AbstractServiceTest.NAMESPACE, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, partition.get(0), partitionColumns.subList(1, 2).toArray(new SchemaColumn[1]), Arrays.asList(partition.get(1)).toArray(new String[1]), AbstractServiceTest.DATA_VERSION);
        // Create a business object data entity.
        BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partition.get(0), Arrays.asList(partition.get(1)), AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        // Create an "available" storage unit with a storage directory path.
        result.add(storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix));
    }
    return result;
}

Example 99 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForFinalizeRestoreTesting.

/**
 * Create and persist database entities required for the finalize restore testing.
 *
 * @param businessObjectDataKey the business object data key
 * @param storageName the storage name
 * @param s3BucketName the S3 bucket name
 * @param s3StorageUnitStatus the storage unit status
 *
 * @return the business object data entity
 */
public BusinessObjectDataEntity createDatabaseEntitiesForFinalizeRestoreTesting(BusinessObjectDataKey businessObjectDataKey, String storageName, String s3BucketName, String s3StorageUnitStatus) {
    // Create
    // Create and persist a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.BDATA_STATUS);
    // Create and persist an S3 storage entity.
    StorageEntity storageEntity;
    if (s3BucketName != null) {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), s3BucketName), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    } else {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, StoragePlatformEntity.S3);
    }
    // Create and persist a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, s3StorageUnitStatus, AbstractServiceTest.NO_STORAGE_DIRECTORY_PATH);
    // Get the expected S3 key prefix for the business object data key.
    String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(businessObjectDataKey, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.PARTITION_KEY, AbstractServiceTest.NO_SUB_PARTITION_KEYS);
    // Create and add storage file entities to the storage unit.
    for (String relativeFilePath : AbstractServiceTest.LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/%s", s3KeyPrefix, relativeFilePath), AbstractServiceTest.FILE_SIZE_1_KB, AbstractServiceTest.ROW_COUNT);
    }
    // Return the business object data entity.
    return businessObjectDataEntity;
}

Example 100 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForBusinessObjectDataDdlTesting.

/**
 * Creates relative database entities required for the unit tests.
 */
public void createDatabaseEntitiesForBusinessObjectDataDdlTesting(String businessObjectFormatFileType, String partitionKey, String partitionKeyGroupName, int partitionColumnPosition, List<String> partitionValues, List<String> subPartitionValues, String schemaDelimiterCharacter, String schemaEscapeCharacter, String schemaNullValue, List<SchemaColumn> schemaColumns, List<SchemaColumn> partitionColumns, boolean replaceUnderscoresWithHyphens, String customDdlName, boolean generateStorageFileEntities, boolean allowDuplicateBusinessObjectData) {
    // Create a business object format entity if it does not exist.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, businessObjectFormatFileType, AbstractServiceTest.FORMAT_VERSION));
    if (businessObjectFormatEntity == null) {
        businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, businessObjectFormatFileType, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FORMAT_DESCRIPTION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, partitionKey, partitionKeyGroupName, AbstractServiceTest.NO_ATTRIBUTES, schemaDelimiterCharacter, schemaEscapeCharacter, schemaNullValue, schemaColumns, partitionColumns);
    }
    if (StringUtils.isNotBlank(customDdlName)) {
        boolean partitioned = (partitionColumns != null);
        customDdlDaoTestHelper.createCustomDdlEntity(businessObjectFormatEntity, customDdlName, customDdlServiceTestHelper.getTestCustomDdl(partitioned));
    }
    // Create S3 storages with the relative "bucket.name" attribute configured.
    StorageEntity storageEntity1 = storageDao.getStorageByName(AbstractServiceTest.STORAGE_NAME);
    if (storageEntity1 == null) {
        storageEntity1 = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    }
    StorageEntity storageEntity2 = storageDao.getStorageByName(AbstractServiceTest.STORAGE_NAME_2);
    if (storageEntity2 == null) {
        storageEntity2 = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME_2, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME_2), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    }
    // Create business object data for each partition value.
    for (String partitionValue : partitionValues) {
        BusinessObjectDataEntity businessObjectDataEntity;
        // Create a business object data instance for the specified partition value.
        if (partitionColumnPosition == BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION) {
            businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, businessObjectFormatFileType, AbstractServiceTest.FORMAT_VERSION, partitionValue, subPartitionValues, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        } else {
            List<String> testSubPartitionValues = new ArrayList<>(subPartitionValues);
            // Please note that the second partition column is located at index 0.
            testSubPartitionValues.set(partitionColumnPosition - 2, partitionValue);
            businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, businessObjectFormatFileType, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.PARTITION_VALUE, testSubPartitionValues, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        }
        // Get the expected S3 key prefix.
        String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), AbstractServiceTest.STORAGE_NAME);
        // Check if we need to create the relative storage units.
        if (AbstractServiceTest.STORAGE_1_AVAILABLE_PARTITION_VALUES.contains(partitionValue) || Hive13DdlGenerator.NO_PARTITIONING_PARTITION_VALUE.equals(partitionValue)) {
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity1, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, AbstractServiceTest.NO_STORAGE_DIRECTORY_PATH);
            // Please note that is n! - thus we want to keep the number of partition levels small.
            if (generateStorageFileEntities) {
                storageFileDaoTestHelper.createStorageFileEntities(storageUnitEntity, s3KeyPrefix, partitionColumns, subPartitionValues, replaceUnderscoresWithHyphens);
            } else // Add storage directory path value to the storage unit, since we have no storage files generated.
            {
                storageUnitEntity.setDirectoryPath(s3KeyPrefix);
            }
        }
        if (AbstractServiceTest.STORAGE_2_AVAILABLE_PARTITION_VALUES.contains(partitionValue) && (allowDuplicateBusinessObjectData || !AbstractServiceTest.STORAGE_1_AVAILABLE_PARTITION_VALUES.contains(partitionValue))) {
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity2, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, AbstractServiceTest.NO_STORAGE_DIRECTORY_PATH);
            // Please note that is n! - thus we want to keep the number of partition levels small.
            if (generateStorageFileEntities) {
                storageFileDaoTestHelper.createStorageFileEntities(storageUnitEntity, s3KeyPrefix, partitionColumns, subPartitionValues, replaceUnderscoresWithHyphens);
            } else // Add storage directory path value to the storage unit, since we have no storage files generated.
            {
                storageUnitEntity.setDirectoryPath(s3KeyPrefix);
            }
        }
    }
}