Examples with StorageEntity org.finra.herd.model.jpa.StorageEntity used on opensource projects

Example 96 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getBusinessObjectDataStorageUnitCreateRequest.

public BusinessObjectDataStorageUnitCreateRequest getBusinessObjectDataStorageUnitCreateRequest() {
    // Create business object data and storage entities.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(false);
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(AbstractServiceTest.BDATA_STATUS, AbstractServiceTest.DESCRIPTION, AbstractServiceTest.BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, AbstractServiceTest.PARTITION_VALUE, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, businessObjectDataStatusEntity.getCode());
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity();
    // Create a business object data storage unit key.
    BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey = new BusinessObjectDataStorageUnitKey(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode(), businessObjectFormatEntity.getBusinessObjectDefinition().getName(), businessObjectFormatEntity.getUsage(), businessObjectFormatEntity.getFileType().getCode(), businessObjectFormatEntity.getBusinessObjectFormatVersion(), businessObjectDataEntity.getPartitionValue(), AbstractServiceTest.NO_SUBPARTITION_VALUES, businessObjectDataEntity.getVersion(), storageEntity.getName());
    // Create and return a business object data storage unit create request.
    return new BusinessObjectDataStorageUnitCreateRequest(businessObjectDataStorageUnitKey, AbstractServiceTest.NO_STORAGE_DIRECTORY, getStorageFiles(), AbstractServiceTest.NO_DISCOVER_STORAGE_FILES);
}

Example 97 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForBusinessObjectDataDdlTesting.

/**
 * Creates and persists database entities required for generate business object data ddl testing.
 */
public StorageUnitEntity createDatabaseEntitiesForBusinessObjectDataDdlTesting(String partitionValue, String s3KeyPrefix) {
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, "DATE", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.COLUMN_NAME, "NUMBER", AbstractServiceTest.COLUMN_SIZE, AbstractServiceTest.NO_COLUMN_REQUIRED, AbstractServiceTest.COLUMN_DEFAULT_VALUE, AbstractServiceTest.COLUMN_DESCRIPTION));
    // Use the first column as a partition column.
    List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 1);
    // Create a business object format entity with the schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FORMAT_DESCRIPTION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, AbstractServiceTest.NO_PARTITION_KEY_GROUP, AbstractServiceTest.NO_ATTRIBUTES, AbstractServiceTest.SCHEMA_DELIMITER_PIPE, AbstractServiceTest.SCHEMA_ESCAPE_CHARACTER_BACKSLASH, AbstractServiceTest.SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
    if (partitionValue != null) {
        // Create a business object data entity.
        BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partitionValue, AbstractServiceTest.NO_SUBPARTITION_VALUES, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        // Create an S3 storage entity.
        StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
        // Create a storage unit with a storage directory path.
        return storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix);
    }
    return null;
}

Example 98 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method validateBusinessObjectData.

/**
 * Validates business object data against specified arguments and expected (hard coded) test values.
 *
 * @param request the business object data create request
 * @param expectedBusinessObjectDataVersion the expected business object data version
 * @param expectedLatestVersion the expected business
 * @param actualBusinessObjectData the business object data availability object instance to be validated
 */
public void validateBusinessObjectData(BusinessObjectDataCreateRequest request, Integer expectedBusinessObjectDataVersion, Boolean expectedLatestVersion, BusinessObjectData actualBusinessObjectData) {
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(org.apache.commons.lang3.StringUtils.isNotBlank(request.getNamespace()) ? request.getNamespace() : AbstractServiceTest.NAMESPACE, request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()));
    List<String> expectedSubPartitionValues = CollectionUtils.isEmpty(request.getSubPartitionValues()) ? new ArrayList<>() : request.getSubPartitionValues();
    String expectedStatusCode = org.apache.commons.lang3.StringUtils.isNotBlank(request.getStatus()) ? request.getStatus() : BusinessObjectDataStatusEntity.VALID;
    StorageUnitCreateRequest storageUnitCreateRequest = request.getStorageUnits().get(0);
    StorageEntity storageEntity = storageDao.getStorageByName(storageUnitCreateRequest.getStorageName());
    String expectedStorageDirectoryPath = storageUnitCreateRequest.getStorageDirectory() != null ? storageUnitCreateRequest.getStorageDirectory().getDirectoryPath() : null;
    List<StorageFile> expectedStorageFiles = CollectionUtils.isEmpty(storageUnitCreateRequest.getStorageFiles()) ? null : storageUnitCreateRequest.getStorageFiles();
    List<Attribute> expectedAttributes = CollectionUtils.isEmpty(request.getAttributes()) ? new ArrayList<>() : request.getAttributes();
    validateBusinessObjectData(businessObjectFormatEntity, request.getPartitionValue(), expectedSubPartitionValues, expectedBusinessObjectDataVersion, expectedLatestVersion, expectedStatusCode, storageEntity.getName(), expectedStorageDirectoryPath, expectedStorageFiles, expectedAttributes, actualBusinessObjectData);
}

Example 99 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class FileUploadCleanupServiceTest method createTestDatabaseEntities.

private void createTestDatabaseEntities(BusinessObjectDataKey businessObjectDataKey, String storageName, String bucketName, String storageFilePath, int createdOnTimestampMinutesOffset) throws Exception {
    // Create a storage entity.
    StorageEntity storageEntity = createTestStorageEntity(storageName, bucketName);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, true, BDATA_STATUS);
    // Apply the offset in minutes to createdOn value.
    businessObjectDataEntity.setCreatedOn(new Timestamp(businessObjectDataEntity.getCreatedOn().getTime() - createdOnTimestampMinutesOffset * 60 * 1000));
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
    herdDao.saveAndRefresh(businessObjectDataEntity);
}

Example 100 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataInvalidateUnregisteredHelper method invalidateUnregisteredBusinessObjectData.

/**
 * Compares objects registered vs what exists in S3. Registers objects in INVALID status for data that are not registered but exist in S3. S3 objects are
 * identified by herd's S3 key prefix.
 *
 * @param businessObjectDataInvalidateUnregisteredRequest the request
 *
 * @return response, optionally containing the data that have been registered.
 */
public BusinessObjectDataInvalidateUnregisteredResponse invalidateUnregisteredBusinessObjectData(BusinessObjectDataInvalidateUnregisteredRequest businessObjectDataInvalidateUnregisteredRequest) {
    // Validate request
    validateRequest(businessObjectDataInvalidateUnregisteredRequest);
    // Trim
    trimRequest(businessObjectDataInvalidateUnregisteredRequest);
    // Validate format exists
    // Get format
    BusinessObjectFormatEntity businessObjectFormatEntity = getBusinessObjectFormatEntity(businessObjectDataInvalidateUnregisteredRequest);
    // Validate storage exists
    // Get storage by name
    String storageName = businessObjectDataInvalidateUnregisteredRequest.getStorageName();
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageName);
    // Validate that storage platform is S3
    if (!StoragePlatformEntity.S3.equals(storageEntity.getStoragePlatform().getName())) {
        throw new IllegalArgumentException("The specified storage '" + storageName + "' is not an S3 storage platform.");
    }
    // Get data with latest version
    BusinessObjectDataEntity latestBusinessObjectDataEntity = getLatestBusinessObjectDataEntity(businessObjectDataInvalidateUnregisteredRequest);
    Integer latestBusinessObjectDataVersion = getBusinessObjectDataVersion(latestBusinessObjectDataEntity);
    // List data which are not registered, but exists in S3
    List<BusinessObjectDataKey> unregisteredBusinessObjectDataKeys = getUnregisteredBusinessObjectDataKeys(businessObjectDataInvalidateUnregisteredRequest, storageEntity, businessObjectFormatEntity, latestBusinessObjectDataVersion);
    // Register the unregistered data as INVALID
    List<BusinessObjectDataEntity> registeredBusinessObjectDataEntities = registerInvalidBusinessObjectDatas(latestBusinessObjectDataEntity, businessObjectFormatEntity, unregisteredBusinessObjectDataKeys, storageEntity);
    // Fire notifications
    processBusinessObjectDataStatusChangeNotificationEvents(registeredBusinessObjectDataEntities);
    // Create and return response
    return getBusinessObjectDataInvalidateUnregisteredResponse(businessObjectDataInvalidateUnregisteredRequest, registeredBusinessObjectDataEntities);
}