Examples with StorageEntity org.finra.herd.model.jpa.StorageEntity used on opensource projects

Example 46 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLargePartitionValueListWithAutoDiscovery.

@Test
@Ignore
public void testGenerateBusinessObjectDataDdlLargePartitionValueListWithAutoDiscovery() {
    final int PRIMARY_PARTITION_VALUE_LIST_SIZE = 10000;
    final int SECOND_LEVEL_PARTITION_VALUES_PER_BUSINESS_OBJECT_DATA = 1;
    final int STORAGE_FILES_PER_PARTITION = 1;
    // Prepare test data and build a list of partition values to generate business object data DDL for.
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(PARTITION_KEY, "DATE", NO_COLUMN_SIZE, COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(COLUMN_NAME, "NUMBER", COLUMN_SIZE, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(COLUMN_NAME_2, "STRING", NO_COLUMN_SIZE, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, NO_COLUMN_DESCRIPTION));
    // Use the first two columns as partition columns.
    List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 2);
    // Create a business object format entity with the schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
    // Create an S3 storage entity.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME);
    // Create relative business object data, storage unit, and storage file entities.
    List<String> partitionValues = new ArrayList<>();
    for (int i = 0; i < PRIMARY_PARTITION_VALUE_LIST_SIZE; i++) {
        String partitionValue = String.format("%s-%03d", PARTITION_VALUE, i);
        partitionValues.add(partitionValue);
        // Create a business object data entity.
        BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BusinessObjectDataStatusEntity.VALID);
        // Build an S3 key prefix according to the herd S3 naming convention.
        String s3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, PARTITION_KEY, partitionValue, null, null, DATA_VERSION);
        // Create a storage unit with a storage directory path.
        StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix);
        // Create storage file entities.
        for (int j = 0; j < SECOND_LEVEL_PARTITION_VALUES_PER_BUSINESS_OBJECT_DATA; j++) {
            // Build a storage file directory path that includes the relative second level partition value - needed for auto discovery.
            String storageFileDirectoryPath = String.format("%s/%s=%s-%03d", s3KeyPrefix, COLUMN_NAME, PARTITION_VALUE_2, j);
            for (int k = 0; k < STORAGE_FILES_PER_PARTITION; k++) {
                String storageFilePath = String.format("%s/%03d.data", storageFileDirectoryPath, k);
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
        }
        herdDao.saveAndRefresh(storageUnitEntity);
    }
    // Retrieve business object data ddl for the entire list of partition values.
    BusinessObjectDataDdl businessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, partitionValues, NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, NO_INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
    // Validate the results.
    assertNotNull(businessObjectDataDdl);
}

Example 47 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class AbstractHerdDao method getMaximumBusinessObjectDataVersionSubQuery.

/**
 * TODO This method may be bdata specific. Consider creating new abstract class to group all bdata related DAO. Builds a sub-query to select the maximum
 * business object data version.
 *
 * @param builder the criteria builder
 * @param criteria the criteria query
 * @param businessObjectDataEntity the business object data entity that appears in the from clause of the main query
 * @param businessObjectFormatEntity the business object format entity that appears in the from clause of the main query
 * @param businessObjectDataStatus the business object data status
 * @param storageNames the list of storage names where the business object data storage units should be looked for (case-insensitive)
 * @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
 * @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
 * the storage platform type is specified
 * @param selectOnlyAvailableStorageUnits specifies if only available storage units will be selected or any storage units regardless of their status
 *
 * @return the sub-query to select the maximum business object data version
 */
protected Subquery<Integer> getMaximumBusinessObjectDataVersionSubQuery(CriteriaBuilder builder, CriteriaQuery<?> criteria, From<?, BusinessObjectDataEntity> businessObjectDataEntity, From<?, BusinessObjectFormatEntity> businessObjectFormatEntity, String businessObjectDataStatus, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, boolean selectOnlyAvailableStorageUnits) {
    // Business object data version is not specified, so get the latest one in the specified storage.
    Subquery<Integer> subQuery = criteria.subquery(Integer.class);
    // The criteria root is the business object data.
    Root<BusinessObjectDataEntity> subBusinessObjectDataEntity = subQuery.from(BusinessObjectDataEntity.class);
    // Join to the other tables we can filter on.
    Join<BusinessObjectDataEntity, StorageUnitEntity> subStorageUnitEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.storageUnits);
    Join<StorageUnitEntity, StorageEntity> subStorageEntity = subStorageUnitEntity.join(StorageUnitEntity_.storage);
    Join<StorageEntity, StoragePlatformEntity> subStoragePlatformEntity = subStorageEntity.join(StorageEntity_.storagePlatform);
    Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> subBusinessObjectFormatEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
    Join<StorageUnitEntity, StorageUnitStatusEntity> subStorageUnitStatusEntity = subStorageUnitEntity.join(StorageUnitEntity_.status);
    // Add a standard restriction on business object format.
    Predicate subQueryRestriction = builder.equal(subBusinessObjectFormatEntity, businessObjectFormatEntity);
    // Create and add standard restrictions on primary and sub-partition values.
    subQueryRestriction = builder.and(subQueryRestriction, getQueryRestrictionOnPartitionValues(builder, subBusinessObjectDataEntity, businessObjectDataEntity));
    // If specified, create and add a standard restriction on business object data status.
    if (businessObjectDataStatus != null) {
        Join<BusinessObjectDataEntity, BusinessObjectDataStatusEntity> subBusinessObjectDataStatusEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.status);
        subQueryRestriction = builder.and(subQueryRestriction, builder.equal(builder.upper(subBusinessObjectDataStatusEntity.get(BusinessObjectDataStatusEntity_.code)), businessObjectDataStatus.toUpperCase()));
    }
    // Create and add a standard restriction on storage.
    subQueryRestriction = builder.and(subQueryRestriction, getQueryRestrictionOnStorage(builder, subStorageEntity, subStoragePlatformEntity, storageNames, storagePlatformType, excludedStoragePlatformType));
    // If specified, add a restriction on storage unit status availability flag.
    if (selectOnlyAvailableStorageUnits) {
        subQueryRestriction = builder.and(subQueryRestriction, builder.isTrue(subStorageUnitStatusEntity.get(StorageUnitStatusEntity_.available)));
    }
    subQuery.select(builder.max(subBusinessObjectDataEntity.get(BusinessObjectDataEntity_.version))).where(subQueryRestriction);
    return subQuery;
}

Example 48 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class UploadDownloadServiceTest method testDownloadBusinessObjectDefinitionSampleFileUpperCaseParameters.

@Test
public void testDownloadBusinessObjectDefinitionSampleFileUpperCaseParameters() {
    // Create and persist a business object definition entity.
    businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, BDEF_DISPLAY_NAME, businessObjectDefinitionServiceTestHelper.getNewAttributes(), businessObjectDefinitionServiceTestHelper.getTestSampleDataFiles());
    List<SampleDataFile> sampleFileList = businessObjectDefinitionServiceTestHelper.getTestSampleDataFiles();
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(STORAGE_NAME);
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), "testBucketName"));
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_DOWNLOAD_ROLE_ARN), "downloadRole"));
    DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationRequest downloadRequest = new DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationRequest();
    BusinessObjectDefinitionSampleDataFileKey sampleDataFileKey = new BusinessObjectDefinitionSampleDataFileKey();
    sampleDataFileKey.setBusinessObjectDefinitionName(BDEF_NAME);
    sampleDataFileKey.setNamespace(NAMESPACE);
    sampleDataFileKey.setDirectoryPath(sampleFileList.get(0).getDirectoryPath());
    sampleDataFileKey.setFileName(sampleFileList.get(0).getFileName());
    // use the lower case name space and business definition still return the same response
    BusinessObjectDefinitionSampleDataFileKey sampleDataFileKeyUpperCase = new BusinessObjectDefinitionSampleDataFileKey();
    sampleDataFileKeyUpperCase.setBusinessObjectDefinitionName(BDEF_NAME.toUpperCase());
    sampleDataFileKeyUpperCase.setNamespace(NAMESPACE.toUpperCase());
    sampleDataFileKeyUpperCase.setDirectoryPath(sampleFileList.get(0).getDirectoryPath());
    sampleDataFileKeyUpperCase.setFileName(sampleFileList.get(0).getFileName());
    downloadRequest.setBusinessObjectDefinitionSampleDataFileKey(sampleDataFileKeyUpperCase);
    DownloadBusinessObjectDefinitionSampleDataFileSingleInitiationResponse downloadResponse = uploadDownloadService.initiateDownloadSingleSampleFile(downloadRequest);
    assertEquals(downloadResponse.getBusinessObjectDefinitionSampleDataFileKey(), sampleDataFileKey);
    assertNotNull(downloadResponse.getAwsS3BucketName());
    assertNotNull(downloadResponse.getAwsAccessKey());
    assertNotNull(downloadResponse.getAwsSecretKey());
    assertNotNull(downloadResponse.getAwsSessionExpirationTime());
    assertNotNull(downloadResponse.getAwsSessionToken());
    assertNotNull(downloadResponse.getPreSignedUrl());
}

Example 49 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageServiceTest method testUpdateStorageAttributesStorageHasDuplicateAttributes.

@Test
public void testUpdateStorageAttributesStorageHasDuplicateAttributes() {
    // Create and persist a valid storage.
    StorageCreateRequest request = getNewStorageCreateRequest();
    Storage storage = storageService.createStorage(request);
    // Add a duplicate attribute to the storage.
    StorageEntity storageEntity = storageDao.getStorageByName(storage.getName());
    StorageAttributeEntity storageAttributeEntity = new StorageAttributeEntity();
    storageAttributeEntity.setStorage(storageEntity);
    storageAttributeEntity.setName(request.getAttributes().get(0).getName().toUpperCase());
    storageEntity.getAttributes().add(storageAttributeEntity);
    storageDao.saveAndRefresh(storageEntity);
    // Try to update attributes for the storage.
    try {
        storageService.updateStorageAttributes(new StorageKey(storage.getName()), new StorageAttributesUpdateRequest(businessObjectDefinitionServiceTestHelper.getNewAttributes2()));
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found duplicate attribute with name \"%s\" for \"%s\" storage.", request.getAttributes().get(0).getName().toLowerCase(), storage.getName()), e.getMessage());
    }
}

Example 50 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class NotificationEventServiceTest method testProcessBusinessObjectDataRegistrationNotificationEventSyncAssertFireEnabledOnly.

@Test
public void testProcessBusinessObjectDataRegistrationNotificationEventSyncAssertFireEnabledOnly() throws Exception {
    // Create job definition
    JobDefinition jobDefinition = jobDefinitionServiceTestHelper.createJobDefinition(ACTIVITI_XML_LOG_VARIABLES_NO_REGEX_WITH_CLASSPATH);
    List<JobAction> jobActions1 = Arrays.asList(new JobAction(jobDefinition.getNamespace(), jobDefinition.getJobName(), CORRELATION_DATA));
    List<JobAction> jobActions2 = Arrays.asList(new JobAction(jobDefinition.getNamespace(), jobDefinition.getJobName(), CORRELATION_DATA_2));
    // Create a business object format with a schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), schemaColumnDaoTestHelper.getTestPartitionColumns());
    // Create business object data with storage units.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3);
    storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Create and persist a business object data notification registration entity that is enabled.
    notificationRegistrationDaoTestHelper.createBusinessObjectDataNotificationRegistrationEntity(new NotificationRegistrationKey(NAMESPACE, NOTIFICATION_NAME), NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN.name(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, STORAGE_NAME, BDATA_STATUS, null, jobActions1, NotificationRegistrationStatusEntity.ENABLED);
    // Create and persist a business object data notification registration entity that is disabled.
    notificationRegistrationDaoTestHelper.createBusinessObjectDataNotificationRegistrationEntity(new NotificationRegistrationKey(NAMESPACE, NOTIFICATION_NAME_2), NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN.name(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, STORAGE_NAME_2, BDATA_STATUS, null, jobActions2, NotificationRegistrationStatusEntity.DISABLED);
    // Trigger the notification
    List<Object> notificationActions = notificationEventService.processBusinessObjectDataNotificationEventSync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), BDATA_STATUS, null);
    assertEquals(1, notificationActions.size());
    assertEquals(Job.class, notificationActions.get(0).getClass());
    Job job = (Job) notificationActions.get(0);
    List<Parameter> parameters = job.getParameters();
    boolean found = false;
    for (Parameter parameter : parameters) {
        String name = parameter.getName();
        if ("notification_correlationData".equals(name)) {
            assertEquals(CORRELATION_DATA, parameter.getValue());
            found = true;
            break;
        }
    }
    assertTrue(found);
}