Examples with BusinessObjectDataDdl org.finra.herd.model.api.xml.BusinessObjectDataDdl used on opensource projects

Example 26 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceImpl method generateBusinessObjectDataDdlImpl.

/**
 * Retrieves the DDL to initialize the specified type of the database system to perform queries for a range of requested business object data in the
 * specified storage.
 *
 * @param request the business object data DDL request
 * @param skipRequestValidation specifies whether to skip the request validation and trimming
 *
 * @return the business object data DDL information
 */
protected BusinessObjectDataDdl generateBusinessObjectDataDdlImpl(BusinessObjectDataDdlRequest request, boolean skipRequestValidation) {
    // Perform the validation.
    if (!skipRequestValidation) {
        validateBusinessObjectDataDdlRequest(request);
    }
    // Get the business object format entity for the specified parameters and make sure it exists.
    // Please note that when format version is not specified, we should get back the latest format version.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoHelper.getBusinessObjectFormatEntity(new BusinessObjectFormatKey(request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()));
    // Validate that format has schema information.
    Assert.notEmpty(businessObjectFormatEntity.getSchemaColumns(), String.format("Business object format with namespace \"%s\", business object definition name \"%s\", format usage \"%s\", format file type \"%s\"," + " and format version \"%s\" doesn't have schema information.", businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode(), businessObjectFormatEntity.getBusinessObjectDefinition().getName(), businessObjectFormatEntity.getUsage(), businessObjectFormatEntity.getFileType().getCode(), businessObjectFormatEntity.getBusinessObjectFormatVersion()));
    // If it was specified, retrieve the custom DDL and ensure it exists.
    CustomDdlEntity customDdlEntity = null;
    if (StringUtils.isNotBlank(request.getCustomDdlName())) {
        CustomDdlKey customDdlKey = new CustomDdlKey(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode(), businessObjectFormatEntity.getBusinessObjectDefinition().getName(), businessObjectFormatEntity.getUsage(), businessObjectFormatEntity.getFileType().getCode(), businessObjectFormatEntity.getBusinessObjectFormatVersion(), request.getCustomDdlName());
        customDdlEntity = customDdlDaoHelper.getCustomDdlEntity(customDdlKey);
    }
    // Validate that specified storages exist and of a proper storage platform type.
    List<String> storageNames = new ArrayList<>();
    if (StringUtils.isNotBlank(request.getStorageName())) {
        storageNames.add(request.getStorageName());
    }
    if (!CollectionUtils.isEmpty(request.getStorageNames())) {
        storageNames.addAll(request.getStorageNames());
    }
    List<StorageEntity> storageEntities = new ArrayList<>();
    for (String storageName : storageNames) {
        StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageName);
        // Only S3 storage platform is currently supported.
        Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot generate DDL for \"%s\" storage platform.", storageEntity.getStoragePlatform().getName()));
        storageEntities.add(storageEntity);
    }
    // Validate that all storages have S3 bucket name configured.
    Map<StorageEntity, String> s3BucketNames = new HashMap<>();
    for (StorageEntity storageEntity : storageEntities) {
        // Please note that since S3 bucket name attribute value is required we pass a "true" flag.
        String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageEntity, true);
        s3BucketNames.put(storageEntity, s3BucketName);
    }
    // Create and initialize a business object data DDL object instance.
    BusinessObjectDataDdl businessObjectDataDdl = createBusinessObjectDataDdl(request);
    businessObjectDataDdl.setDdl(ddlGeneratorFactory.getDdlGenerator(request.getOutputFormat()).generateCreateTableDdl(request, businessObjectFormatEntity, customDdlEntity, storageNames, storageEntities, s3BucketNames));
    return businessObjectDataDdl;
}

Example 27 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class GenerateBusinessObjectDataDdl method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    String contentTypeString = activitiHelper.getRequiredExpressionVariableAsString(contentType, execution, "ContentType").trim();
    String requestString = activitiHelper.getRequiredExpressionVariableAsString(businessObjectDataDdlRequest, execution, "businessObjectDataDdlRequest").trim();
    BusinessObjectDataDdlRequest request = getRequestObject(contentTypeString, requestString, BusinessObjectDataDdlRequest.class);
    // Generate Ddl.
    BusinessObjectDataDdl businessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    setTaskWorkflowVariable(execution, VARIABLE_DDL, businessObjectDataDdl.getDdl());
}

Example 28 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionValid.

@Test
public void testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionValid() {
    // 
    List<List<String>> testPartitions = Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1), Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_2));
    // Create two VALID sub-partitions both with "available" storage units in a non-Glacier storage.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(testPartitions);
    // Retrieve business object data DDL with "IncludeAllRegisteredSubPartitions" option enabled.
    BusinessObjectDataDdl resultBusinessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
    // Validate the response object. Both sub-partitions should be present in the generated DDL.
    assertEquals(new BusinessObjectDataDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdlTwoPartitionLevels(testPartitions)), resultBusinessObjectDataDdl);
}

Example 29 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLatestFormatVersionDataNotAvailableInStorage.

@Test
public void testGenerateBusinessObjectDataDdlLatestFormatVersionDataNotAvailableInStorage() {
    // Create database entities for two versions of a business object format.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1 + SUBPARTITION_VALUES.size());
    BusinessObjectFormatEntity businessObjectFormatV0Entity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, NO_LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
    BusinessObjectFormatEntity businessObjectFormatV1Entity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
    // Create two storage entities.
    StorageEntity storage1Entity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    StorageEntity storage2Entity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2);
    // Register initial version of the business object data for initial format version in both storages.
    BusinessObjectDataEntity businessObjectDataV0V0Entity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatV0Entity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataV0V0Entity), storage1Entity.getName());
    for (StorageEntity storageEntity : Arrays.asList(storage1Entity, storage2Entity)) {
        StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataV0V0Entity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
        storageUnitEntity.setDirectoryPath(s3KeyPrefix);
        herdDao.saveAndRefresh(storageUnitEntity);
    }
    herdDao.saveAndRefresh(businessObjectDataV0V0Entity);
    // Register initial version of the business object data for second format version, but only in the second storage.
    BusinessObjectDataEntity businessObjectDataV1V0Entity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatV1Entity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataV1V0Entity), storage2Entity.getName());
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storage2Entity, businessObjectDataV1V0Entity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    storageUnitEntity.setDirectoryPath(s3KeyPrefix);
    herdDao.saveAndRefresh(storageUnitEntity);
    herdDao.saveAndRefresh(businessObjectDataV1V0Entity);
    // Retrieve business object data ddl for the first storage without specifying business object format version.
    BusinessObjectDataDdlRequest request = new BusinessObjectDataDdlRequest();
    request.setNamespace(NAMESPACE);
    request.setBusinessObjectDefinitionName(BDEF_NAME);
    request.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    request.setBusinessObjectFormatFileType(FileTypeEntity.TXT_FILE_TYPE);
    request.setBusinessObjectFormatVersion(null);
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    request.setPartitionValueFilter(partitionValueFilter);
    partitionValueFilter.setPartitionKey(FIRST_PARTITION_COLUMN_NAME);
    partitionValueFilter.setPartitionValues(Arrays.asList(PARTITION_VALUE));
    request.setBusinessObjectDataVersion(INITIAL_DATA_VERSION);
    request.setStorageName(StorageEntity.MANAGED_STORAGE);
    request.setOutputFormat(BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL);
    request.setTableName(TABLE_NAME);
    request.setCustomDdlName(null);
    request.setIncludeDropTableStatement(true);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    assertNotNull(resultDdl);
}

Example 30 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoDropTable.

@Test
public void testGenerateBusinessObjectDataDdlNoDropTable() {
    // Prepare test data without custom ddl.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), schemaColumnDaoTestHelper.getTestPartitionColumns(), false, CUSTOM_DDL_NAME, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl request without drop table statement.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    request.setIncludeDropTableStatement(false);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, false, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}