Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 66 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceImpl method createSchemaColumnEntities.

/**
 * Creates the schema column entities.
 *
 * @param schemaColumns the list of schema columns (for either the data columns or the partition columns).
 * @param isPartitionList A flag to specify whether the list of schema columns is a data column or a partition column list.
 * @param schemaColumnEntityList the list of schema column entities we're creating. This method will add a new one to the list if it hasn't been created
 * before.
 * @param schemaColumnEntityMap a map of schema column entity names to the schema column entity. This is used so we don't have to keep searching the
 * schemaColumnEntityList which is less efficient.
 * @param businessObjectFormatEntity the business object format entity to associated each newly created schema column entity with.
 */
private void createSchemaColumnEntities(List<SchemaColumn> schemaColumns, boolean isPartitionList, Collection<SchemaColumnEntity> schemaColumnEntityList, Map<String, SchemaColumnEntity> schemaColumnEntityMap, BusinessObjectFormatEntity businessObjectFormatEntity) {
    // Create the relative database entities if schema columns are specified.
    if (!CollectionUtils.isEmpty(schemaColumns)) {
        // Initialize a position counter since the order we encounter the schema columns is the order we will set on the entity.
        int position = 1;
        // Loop through each schema column in the list.
        for (SchemaColumn schemaColumn : schemaColumns) {
            // See if we created the schema column already. If not, create it and add it to the map and the collection.
            SchemaColumnEntity schemaColumnEntity = schemaColumnEntityMap.get(schemaColumn.getName());
            if (schemaColumnEntity == null) {
                schemaColumnEntity = createSchemaColumnEntity(schemaColumn, businessObjectFormatEntity);
                schemaColumnEntityList.add(schemaColumnEntity);
                schemaColumnEntityMap.put(schemaColumn.getName(), schemaColumnEntity);
            }
            // Set the position or partition level depending on the type of object we're processing.
            if (isPartitionList) {
                schemaColumnEntity.setPartitionLevel(position++);
            } else {
                schemaColumnEntity.setPosition(position++);
            }
        }
    }
}

Example 67 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class Hive13DdlGeneratorTest method getPartitionColumns.

private List<SchemaColumn> getPartitionColumns(List<String> partitionColumnNames) {
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    for (String partitionColumnName : partitionColumnNames) {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumns.add(schemaColumn);
        schemaColumn.setName(partitionColumnName);
    }
    return schemaColumns;
}

Example 68 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class Hive13DdlGeneratorTest method testGetHivePartitions.

@Test
public void testGetHivePartitions() {
    // Create a test business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, BDATA_STATUS);
    List<SchemaColumn> autoDiscoverableSubPartitionColumns;
    List<String> storageFilePaths;
    List<HivePartitionDto> expectedHivePartitions;
    List<HivePartitionDto> resultHivePartitions;
    // Get business object data key.
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity);
    // No storage files.
    autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2"));
    storageFilePaths = new ArrayList<>();
    expectedHivePartitions = new ArrayList<>();
    resultHivePartitions = hive13DdlGenerator.getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME);
    assertEquals(expectedHivePartitions, resultHivePartitions);
    // Single level partitioning.
    autoDiscoverableSubPartitionColumns = new ArrayList<>();
    storageFilePaths = getStorageFilePaths(Arrays.asList("/file1.dat", "/file2.dat"));
    expectedHivePartitions = Arrays.asList(HivePartitionDto.builder().withPath("").withPartitionValues(Arrays.asList(PARTITION_VALUE)).build());
    resultHivePartitions = hive13DdlGenerator.getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME);
    assertEquals(expectedHivePartitions, resultHivePartitions);
    // Test that we match column names in storage file paths ignoring the case.
    autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2"));
    storageFilePaths = getStorageFilePaths(Arrays.asList("/COLUMN1=111/COLUMN2=222/file.dat", "/column1=aa/column2=bb/"));
    expectedHivePartitions = Arrays.asList(HivePartitionDto.builder().withPath("/COLUMN1=111/COLUMN2=222").withPartitionValues(Arrays.asList(PARTITION_VALUE, "111", "222")).build(), HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build());
    resultHivePartitions = hive13DdlGenerator.getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME);
    assertEquals(expectedHivePartitions, resultHivePartitions);
}

Example 69 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class NotificationEventServiceImpl method processStorageUnitNotifications.

private List<Object> processStorageUnitNotifications(String notificationEventType, List<StorageUnitNotificationRegistrationEntity> storageUnitNotificationRegistrationEntities, BusinessObjectData businessObjectData, String storageName, String newStorageUnitStatus, String oldStorageUnitStatus) {
    List<Object> notificationActions = new ArrayList<>();
    // Build a list of partition value that includes primary and sub-partition values, if any are specified in the business object data key.
    List<String> partitionValues = businessObjectDataHelper.getPrimaryAndSubPartitionValues(businessObjectData);
    // Get a list of partition columns from the associated business object format.
    List<String> partitionColumnNames = null;
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(new BusinessObjectFormatKey(businessObjectData.getNamespace(), businessObjectData.getBusinessObjectDefinitionName(), businessObjectData.getBusinessObjectFormatUsage(), businessObjectData.getBusinessObjectFormatFileType(), businessObjectData.getBusinessObjectFormatVersion()));
    if (businessObjectFormatEntity != null) {
        // Get business object format model object to directly access schema columns and partitions.
        BusinessObjectFormat businessObjectFormat = businessObjectFormatHelper.createBusinessObjectFormatFromEntity(businessObjectFormatEntity);
        // Proceed only if this format has schema with partition columns specified.
        if (businessObjectFormat.getSchema() != null && !CollectionUtils.isEmpty(businessObjectFormat.getSchema().getPartitions())) {
            // Do not provide more partition column names than there are primary and
            // sub-partition values that this business object data is registered with.
            partitionColumnNames = new ArrayList<>();
            List<SchemaColumn> partitionColumns = businessObjectFormat.getSchema().getPartitions();
            for (int i = 0; i < Math.min(partitionValues.size(), partitionColumns.size()); i++) {
                partitionColumnNames.add(partitionColumns.get(i).getName());
            }
        }
    }
    for (StorageUnitNotificationRegistrationEntity storageUnitNotificationRegistration : storageUnitNotificationRegistrationEntities) {
        // Retrieve the job notification actions needed to be triggered.
        for (NotificationActionEntity notificationActionEntity : storageUnitNotificationRegistration.getNotificationActions()) {
            // Trigger the job action.
            if (notificationActionEntity instanceof NotificationJobActionEntity) {
                NotificationJobActionEntity notificationJobActionEntity = (NotificationJobActionEntity) notificationActionEntity;
                StorageUnitNotificationEventParamsDto notificationEventParams = new StorageUnitNotificationEventParamsDto();
                notificationEventParams.setStorageUnitNotificationRegistration(storageUnitNotificationRegistration);
                notificationEventParams.setNotificationJobAction(notificationJobActionEntity);
                notificationEventParams.setEventType(notificationEventType);
                notificationEventParams.setBusinessObjectData(businessObjectData);
                notificationEventParams.setPartitionColumnNames(partitionColumnNames);
                notificationEventParams.setStorageName(storageName);
                notificationEventParams.setPartitionValues(partitionValues);
                notificationEventParams.setNewStorageUnitStatus(newStorageUnitStatus);
                notificationEventParams.setOldStorageUnitStatus(oldStorageUnitStatus);
                notificationActions.add(triggerNotificationAction(NotificationTypeEntity.NOTIFICATION_TYPE_STORAGE_UNIT, notificationEventType, notificationEventParams));
            }
        }
    }
    return notificationActions;
}

Example 70 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class RelationalTableRegistrationServiceImpl method createRelationalTableRegistrationImpl.

/**
 * Creates a new relational table registration.
 *
 * @param relationalTableRegistrationCreateRequest the relational table registration create request
 * @param appendToExistingBusinessObjectDefinition boolean flag that determines if the format should be appended to an existing business object definition
 *
 * @return the information for the newly created business object data
 */
BusinessObjectData createRelationalTableRegistrationImpl(RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest, Boolean appendToExistingBusinessObjectDefinition) {
    // Validate the relational table registration create request.
    relationalTableRegistrationHelperService.validateAndTrimRelationalTableRegistrationCreateRequest(relationalTableRegistrationCreateRequest);
    // Get storage attributes required to perform relation table registration.
    // This method also validates database entities per specified relational table registration create request.
    RelationalStorageAttributesDto relationalStorageAttributesDto = relationalTableRegistrationHelperService.getRelationalStorageAttributes(relationalTableRegistrationCreateRequest, appendToExistingBusinessObjectDefinition);
    // Retrieve a list of actual schema columns for the specified relational table.
    // This method uses actual JDBC connection to retrieve a description of table columns.
    List<SchemaColumn> schemaColumns = relationalTableRegistrationHelperService.retrieveRelationalTableColumns(relationalStorageAttributesDto, relationalTableRegistrationCreateRequest.getRelationalSchemaName(), relationalTableRegistrationCreateRequest.getRelationalTableName());
    // Create a new relational table registration and return the information for the newly created business object data.
    return relationalTableRegistrationHelperService.registerRelationalTable(relationalTableRegistrationCreateRequest, schemaColumns, appendToExistingBusinessObjectDefinition);
}