Examples with Attribute org.finra.herd.model.api.xml.Attribute used on opensource projects

Example 6 with Attribute

use of org.finra.herd.model.api.xml.Attribute in project herd by FINRAOS.

the class DownloaderController method createDownloaderOutputManifestDto.

/**
 * Creates a downloader output manifest instance from the business object data.
 *
 * @param businessObjectData the business object data that we need to create the manifest for
 * @param storageUnit the S3 storage unit for this business object data
 * @param s3KeyPrefix the S3 key prefix for this business object data
 *
 * @return the created downloader output manifest instance
 */
protected DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix) {
    DownloaderOutputManifestDto downloaderOutputManifestDto = new DownloaderOutputManifestDto();
    // Populate basic fields.
    downloaderOutputManifestDto.setNamespace(businessObjectData.getNamespace());
    downloaderOutputManifestDto.setBusinessObjectDefinitionName(businessObjectData.getBusinessObjectDefinitionName());
    downloaderOutputManifestDto.setBusinessObjectFormatUsage(businessObjectData.getBusinessObjectFormatUsage());
    downloaderOutputManifestDto.setBusinessObjectFormatFileType(businessObjectData.getBusinessObjectFormatFileType());
    downloaderOutputManifestDto.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    downloaderOutputManifestDto.setPartitionKey(businessObjectData.getPartitionKey());
    downloaderOutputManifestDto.setPartitionValue(businessObjectData.getPartitionValue());
    downloaderOutputManifestDto.setSubPartitionValues(businessObjectData.getSubPartitionValues());
    downloaderOutputManifestDto.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
    downloaderOutputManifestDto.setStorageName(storageUnit.getStorage().getName());
    // Build a list of manifest files with paths relative to the S3 key prefix.
    List<ManifestFile> manifestFiles = new ArrayList<>();
    downloaderOutputManifestDto.setManifestFiles(manifestFiles);
    if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) {
        for (StorageFile storageFile : storageUnit.getStorageFiles()) {
            ManifestFile manifestFile = new ManifestFile();
            manifestFiles.add(manifestFile);
            manifestFile.setFileName(storageFile.getFilePath().replace(s3KeyPrefix, ""));
            manifestFile.setFileSizeBytes(storageFile.getFileSizeBytes());
            manifestFile.setRowCount(storageFile.getRowCount());
        }
    }
    // Populate the attributes.
    HashMap<String, String> attributes = new HashMap<>();
    if (!CollectionUtils.isEmpty(businessObjectData.getAttributes())) {
        for (Attribute attribute : businessObjectData.getAttributes()) {
            attributes.put(attribute.getName(), attribute.getValue());
        }
    }
    downloaderOutputManifestDto.setAttributes(attributes);
    // Populate the business object data parents and children.
    downloaderOutputManifestDto.setBusinessObjectDataParents(businessObjectData.getBusinessObjectDataParents());
    downloaderOutputManifestDto.setBusinessObjectDataChildren(businessObjectData.getBusinessObjectDataChildren());
    return downloaderOutputManifestDto;
}

Example 7 with Attribute

use of org.finra.herd.model.api.xml.Attribute in project herd by FINRAOS.

the class StorageHelperTest method testGetStorageAttributeValueByNameFromStorageEntity.

@Test
public void testGetStorageAttributeValueByNameFromStorageEntity() {
    // Create an S3 storage entity.
    List<Attribute> attributes = new ArrayList<>();
    attributes.add(new Attribute(ATTRIBUTE_NAME_1_MIXED_CASE, ATTRIBUTE_VALUE_1));
    attributes.add(new Attribute(ATTRIBUTE_NAME_2_MIXED_CASE, BLANK_TEXT));
    attributes.add(new Attribute(ATTRIBUTE_NAME_3_MIXED_CASE, null));
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, attributes);
    // Retrieve optional attribute values.
    Assert.assertEquals(ATTRIBUTE_VALUE_1, storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_1_MIXED_CASE, storageEntity, false));
    Assert.assertEquals(BLANK_TEXT, storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_2_MIXED_CASE, storageEntity, false));
    Assert.assertNull(storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_3_MIXED_CASE, storageEntity, false));
    // Validate case insensitivity.
    Assert.assertEquals(ATTRIBUTE_VALUE_1, storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_1_MIXED_CASE.toUpperCase(), storageEntity, false));
    Assert.assertEquals(ATTRIBUTE_VALUE_1, storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_1_MIXED_CASE.toLowerCase(), storageEntity, false));
    // Retrieve a required attribute value.
    Assert.assertEquals(ATTRIBUTE_VALUE_1, storageHelper.getStorageAttributeValueByName(ATTRIBUTE_NAME_1_MIXED_CASE, storageEntity, true));
    // Try to retrieve a missing required attribute values when
    // - attribute does not exist
    // - attribute exists with a blank text value
    // - attribute exists with a null value
    String attributeNoExist = "I_DO_NOT_EXIST";
    for (String attributeName : Arrays.asList(attributeNoExist, ATTRIBUTE_NAME_2_MIXED_CASE, ATTRIBUTE_NAME_3_MIXED_CASE)) {
        try {
            storageHelper.getStorageAttributeValueByName(attributeName, storageEntity, true);
        } catch (IllegalStateException e) {
            if (attributeName.equals(attributeNoExist)) {
                Assert.assertEquals(String.format("Attribute \"%s\" for \"%s\" storage must be configured.", attributeName, STORAGE_NAME), e.getMessage());
            } else {
                Assert.assertEquals(String.format("Attribute \"%s\" for \"%s\" storage must have a value that is not blank.", attributeName, STORAGE_NAME), e.getMessage());
            }
        }
    }
}

Example 8 with Attribute

use of org.finra.herd.model.api.xml.Attribute in project herd by FINRAOS.

the class MockHttpClientOperationsImpl method buildGetBusinessObjectDataResponse.

/**
 * Builds a business object data response.
 *
 * @param response the response.
 * @param uri the URI of the incoming request.
 *
 * @throws JAXBException if a JAXB error occurred.
 */
private void buildGetBusinessObjectDataResponse(MockCloseableHttpResponse response, URI uri) throws JAXBException {
    Pattern pattern = Pattern.compile("/herd-app/rest/businessObjectData/namespaces/(.*)/businessObjectDefinitionNames/(.*)/businessObjectFormatUsages/(.*)" + "/businessObjectFormatFileTypes/(.*).*");
    Matcher matcher = pattern.matcher(uri.getPath());
    if (matcher.find()) {
        BusinessObjectData businessObjectData = new BusinessObjectData();
        businessObjectData.setNamespace(matcher.group(1));
        businessObjectData.setBusinessObjectDefinitionName(matcher.group(2));
        businessObjectData.setBusinessObjectFormatUsage(matcher.group(3));
        businessObjectData.setBusinessObjectFormatFileType(matcher.group(4));
        businessObjectData.setPartitionValue("2014-01-31");
        businessObjectData.setPartitionKey("PROCESS_DATE");
        businessObjectData.setAttributes(new ArrayList<Attribute>());
        businessObjectData.setBusinessObjectFormatVersion(0);
        businessObjectData.setLatestVersion(true);
        businessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
        List<StorageUnit> storageUnits = new ArrayList<>();
        businessObjectData.setStorageUnits(storageUnits);
        StorageUnit storageUnit = new StorageUnit();
        storageUnits.add(storageUnit);
        storageUnit.setStorage(getNewStorage(StorageEntity.MANAGED_STORAGE));
        List<StorageFile> storageFiles = new ArrayList<>();
        storageUnit.setStorageFiles(storageFiles);
        storageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
        List<String> localFiles = Arrays.asList("foo1.dat", "Foo2.dat", "FOO3.DAT", "folder/foo3.dat", "folder/foo2.dat", "folder/foo1.dat");
        for (String filename : localFiles) {
            StorageFile storageFile = new StorageFile();
            storageFiles.add(storageFile);
            storageFile.setFilePath(businessObjectData.getNamespace().toLowerCase().replace('_', '-') + "/exchange-a/" + businessObjectData.getBusinessObjectFormatUsage().toLowerCase().replace('_', '-') + "/" + businessObjectData.getBusinessObjectFormatFileType().toLowerCase().replace('_', '-') + "/" + businessObjectData.getBusinessObjectDefinitionName().toLowerCase().replace('_', '-') + "/frmt-v" + businessObjectData.getBusinessObjectFormatVersion() + "/data-v" + businessObjectData.getVersion() + "/" + businessObjectData.getPartitionKey().toLowerCase().replace('_', '-') + "=" + businessObjectData.getPartitionValue() + "/" + filename);
            storageFile.setFileSizeBytes(1024L);
            storageFile.setRowCount(10L);
        }
        businessObjectData.setSubPartitionValues(new ArrayList<String>());
        businessObjectData.setId(1234);
        businessObjectData.setVersion(0);
        response.setEntity(getHttpEntity(businessObjectData));
    }
}

Example 9 with Attribute

use of org.finra.herd.model.api.xml.Attribute in project herd by FINRAOS.

the class StorageServiceImpl method createStorage.

@Override
public Storage createStorage(StorageCreateRequest storageCreateRequest) {
    // Perform validation and trim.
    validateAndTrimStorageCreateRequest(storageCreateRequest);
    // Retrieve storage platform.
    StoragePlatformEntity storagePlatformEntity = storagePlatformHelper.getStoragePlatformEntity(storageCreateRequest.getStoragePlatformName());
    // See if a storage with the specified name already exists.
    StorageEntity storageEntity = storageDao.getStorageByName(storageCreateRequest.getName());
    if (storageEntity != null) {
        throw new AlreadyExistsException(String.format("Storage with name \"%s\" already exists.", storageCreateRequest.getName()));
    }
    // Create a storage entity.
    storageEntity = new StorageEntity();
    storageEntity.setName(storageCreateRequest.getName());
    storageEntity.setStoragePlatform(storagePlatformEntity);
    // Create attributes if they are specified.
    if (!CollectionUtils.isEmpty(storageCreateRequest.getAttributes())) {
        List<StorageAttributeEntity> attributeEntities = new ArrayList<>();
        storageEntity.setAttributes(attributeEntities);
        for (Attribute attribute : storageCreateRequest.getAttributes()) {
            StorageAttributeEntity attributeEntity = new StorageAttributeEntity();
            attributeEntities.add(attributeEntity);
            attributeEntity.setStorage(storageEntity);
            attributeEntity.setName(attribute.getName());
            attributeEntity.setValue(attribute.getValue());
        }
    }
    // Persist the storage entity.
    storageEntity = storageDao.saveAndRefresh(storageEntity);
    // Return the storage information.
    return createStorageFromEntity(storageEntity);
}

Example 10 with Attribute

use of org.finra.herd.model.api.xml.Attribute in project herd by FINRAOS.

the class StorageServiceImpl method updateStorageAttributesHelper.

/**
 * Updates storage attributes.
 *
 * @param storageEntity the storage entity
 * @param attributes the list of attributes
 * @param storageKey the storage key
 */
private void updateStorageAttributesHelper(StorageEntity storageEntity, List<Attribute> attributes, StorageKey storageKey) {
    // Load all existing attribute entities in a map with a "lowercase" attribute name as the key for case insensitivity.
    Map<String, StorageAttributeEntity> existingAttributeEntities = new HashMap<>();
    for (StorageAttributeEntity attributeEntity : storageEntity.getAttributes()) {
        String mapKey = attributeEntity.getName().toLowerCase();
        if (existingAttributeEntities.containsKey(mapKey)) {
            throw new IllegalStateException(String.format("Found duplicate attribute with name \"%s\" for \"%s\" storage.", mapKey, storageKey.getStorageName()));
        }
        existingAttributeEntities.put(mapKey, attributeEntity);
    }
    // Process the list of attributes to determine that storage attribute entities should be created, updated, or deleted.
    List<StorageAttributeEntity> createdAttributeEntities = new ArrayList<>();
    List<StorageAttributeEntity> retainedAttributeEntities = new ArrayList<>();
    if (!CollectionUtils.isEmpty(attributes)) {
        for (Attribute attribute : attributes) {
            // Use a "lowercase" attribute name for case insensitivity.
            String lowercaseAttributeName = attribute.getName().toLowerCase();
            if (existingAttributeEntities.containsKey(lowercaseAttributeName)) {
                // Check if the attribute value needs to be updated.
                StorageAttributeEntity attributeEntity = existingAttributeEntities.get(lowercaseAttributeName);
                if (!StringUtils.equals(attribute.getValue(), attributeEntity.getValue())) {
                    // Update the attribute entity.
                    attributeEntity.setValue(attribute.getValue());
                }
                // Add this entity to the list of attribute entities to be retained.
                retainedAttributeEntities.add(attributeEntity);
            } else {
                // Create a new attribute entity.
                StorageAttributeEntity attributeEntity = new StorageAttributeEntity();
                storageEntity.getAttributes().add(attributeEntity);
                attributeEntity.setStorage(storageEntity);
                attributeEntity.setName(attribute.getName());
                attributeEntity.setValue(attribute.getValue());
                // Add this entity to the list of the newly created attribute entities.
                createdAttributeEntities.add(attributeEntity);
            }
        }
    }
    // Remove any of the currently existing attribute entities that did not get onto the retained entities list.
    storageEntity.getAttributes().retainAll(retainedAttributeEntities);
    // Add all of the newly created attribute entities.
    storageEntity.getAttributes().addAll(createdAttributeEntities);
}