Examples with StorageEntity org.finra.herd.model.jpa.StorageEntity used on opensource projects

Example 41 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorage.

@Test
public void testGetStorageUnitByBusinessObjectDataAndStorage() {
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    StorageEntity storageEntity = storageUnitEntity.getStorage();
    // Test retrieval by entities.
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageEntity));
    // Test retrieval failures.
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(), storageEntity));
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageDaoTestHelper.createStorageEntity()));
}

Example 42 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
    // Get storage entity.
    StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    // Expected business object format file type to Hive file format mapping.
    HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
    businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
        // Prepare test data for the respective business object format file type.
        List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
        String partitionKey = partitionColumns.get(0).getName();
        BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
        for (String partitionValue : UNSORTED_PARTITION_VALUES) {
            BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
            String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
            // Create two storage files.
            for (int i = 0; i < 2; i++) {
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
            herdDao.saveAndRefresh(storageUnitEntity);
            herdDao.saveAndRefresh(businessObjectDataEntity);
        }
        // Retrieve business object format ddl.
        BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
        request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
        BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        // Validate the results.
        String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
        String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
    }
}

Example 43 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class CleanupDestroyedBusinessObjectDataServiceImplTest method createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting.

/**
 * Creates business object data and associated entities required for the cleanup destroyed business object data unit tests.
 */
public BusinessObjectDataStorageUnitKey createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting() {
    List<SchemaColumn> columns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    // Create a business object format entity.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_COMMA, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, columns, partitionColumns);
    // Create S3 storage entity.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    // Create a business object data entity parent.
    BusinessObjectDataEntity businessObjectDataEntityParent = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_3, FORMAT_USAGE_CODE_3, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_3, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
    // Create business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, Lists.newArrayList(businessObjectDataEntityParent));
    // Create a business object data entity child.
    businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, PARTITION_VALUE_2, SUBPARTITION_VALUES_2, DATA_VERSION_2, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, Lists.newArrayList(businessObjectDataEntity));
    // Create and persist a business object data attribute entity.
    businessObjectDataAttributeDaoTestHelper.createBusinessObjectDataAttributeEntity(businessObjectDataEntity, ATTRIBUTE_NAME, ATTRIBUTE_VALUE);
    // Create the storage units.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.DISABLED, STORAGE_DIRECTORY_PATH);
    storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, STORAGE_DIRECTORY_PATH, FILE_SIZE, ROW_COUNT);
    return storageUnitHelper.createStorageUnitKeyFromEntity(storageUnitEntity);
}

Example 44 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceCreateBusinessObjectDataTest method testCreateBusinessObjectDataPreRegistrationAssertDirectoryPathNotRequiredWhenStatusIsPreRegistrationAndDirectoryIsSetInResponse.

@Test
public void testCreateBusinessObjectDataPreRegistrationAssertDirectoryPathNotRequiredWhenStatusIsPreRegistrationAndDirectoryIsSetInResponse() {
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME);
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), "foo"));
    // Create an initial version of the business object data.
    BusinessObjectDataCreateRequest businessObjectDataCreateRequest = businessObjectDataServiceTestHelper.getNewBusinessObjectDataCreateRequest();
    businessObjectDataCreateRequest.setSubPartitionValues(null);
    businessObjectDataCreateRequest.setStatus("UPLOADING");
    businessObjectDataCreateRequest.setStorageUnits(Arrays.asList(new StorageUnitCreateRequest(STORAGE_NAME, null, null, null)));
    BusinessObjectData businessObjectData = businessObjectDataService.createBusinessObjectData(businessObjectDataCreateRequest);
    assertEquals("foo", businessObjectData.getStorageUnits().get(0).getStorageDirectory().getDirectoryPath());
}

Example 45 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceDeleteBusinessObjectDataTest method createTestDatabaseEntities.

/**
 * Create and persist a business object data entity along with the relative storage related entities.
 *
 * @param storageName the storage name
 * @param storagePlatform the storage platform
 * @param directoryPath the directory path for the storage unit entity
 * @param localFiles the list of local files to create relative storage file entities for
 */
private void createTestDatabaseEntities(String storageName, String storagePlatform, String directoryPath, List<String> localFiles) throws Exception {
    // Create and persist a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    // Create an S3 storage entity if it does not exist.
    StorageEntity storageEntity = storageDao.getStorageByName(storageName);
    if (storageEntity == null) {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, storagePlatform, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageDaoTestHelper.getS3ManagedBucketName());
    }
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, directoryPath);
    // Create relative storage file entities.
    for (String fileLocalPath : localFiles) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/%s", directoryPath, fileLocalPath), FILE_SIZE_1_KB, ROW_COUNT_1000);
    }
    herdDao.saveAndRefresh(businessObjectDataEntity);
}