use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.
the class RelationalTableRegistrationServiceTest method testCreateRelationalTableRegistrationWithAppendToExistingBusinessObjectDefinitionSetToTrue.
@Test
public void testCreateRelationalTableRegistrationWithAppendToExistingBusinessObjectDefinitionSetToTrue() {
// Create an existing business object definition.
BusinessObjectDefinitionEntity existingBusinessObjectDefinitionEntity = businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(BDEF_NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, DESCRIPTION, BDEF_DISPLAY_NAME, new ArrayList<>());
// Create database entities required for relational table registration testing.
relationalTableRegistrationServiceTestHelper.createDatabaseEntitiesForRelationalTableRegistrationTesting(STORAGE_NAME);
// Pick one of the in-memory database tables to be registered as a relational table.
String relationalSchemaName = "PUBLIC";
String relationalTableName = BusinessObjectDefinitionEntity.TABLE_NAME.toUpperCase();
// Create a relational table registration create request for a table that is part of the in-memory database setup as part of DAO mocks.
RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest = new RelationalTableRegistrationCreateRequest(BDEF_NAMESPACE, BDEF_NAME, BDEF_DISPLAY_NAME, FORMAT_USAGE_CODE, DATA_PROVIDER_NAME, relationalSchemaName, relationalTableName, STORAGE_NAME);
// Create a relational table registration.
BusinessObjectData businessObjectData = relationalTableRegistrationService.createRelationalTableRegistration(relationalTableRegistrationCreateRequest, APPEND_TO_EXISTING_BUSINESS_OBJECT_DEFINTION_TRUE);
// Create an expected storage unit.
StorageUnit expectedStorageUnit = new StorageUnit();
expectedStorageUnit.setStorage(new Storage(STORAGE_NAME, StoragePlatformEntity.RELATIONAL, relationalTableRegistrationServiceTestHelper.getStorageAttributes()));
expectedStorageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
// Create an expected business object data.
BusinessObjectData expectedBusinessObjectData = new BusinessObjectData();
expectedBusinessObjectData.setId(businessObjectData.getId());
expectedBusinessObjectData.setNamespace(BDEF_NAMESPACE);
expectedBusinessObjectData.setBusinessObjectDefinitionName(BDEF_NAME);
expectedBusinessObjectData.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
expectedBusinessObjectData.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
expectedBusinessObjectData.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
expectedBusinessObjectData.setPartitionValue(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_VALUE);
expectedBusinessObjectData.setSubPartitionValues(new ArrayList<>());
expectedBusinessObjectData.setVersion(INITIAL_DATA_VERSION);
expectedBusinessObjectData.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
expectedBusinessObjectData.setLatestVersion(LATEST_VERSION_FLAG_SET);
expectedBusinessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
expectedBusinessObjectData.setStorageUnits(Collections.singletonList(expectedStorageUnit));
expectedBusinessObjectData.setAttributes(new ArrayList<>());
expectedBusinessObjectData.setBusinessObjectDataParents(new ArrayList<>());
expectedBusinessObjectData.setBusinessObjectDataChildren(new ArrayList<>());
// Validate the response.
assertEquals(expectedBusinessObjectData, businessObjectData);
// Create a business object format key.
BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE, INITIAL_FORMAT_VERSION);
// Retrieve business object format that was created as part of the relational table registration.
BusinessObjectFormat businessObjectFormat = businessObjectFormatService.getBusinessObjectFormat(businessObjectFormatKey);
// Create an expected schema.
Schema expectedSchema = new Schema();
expectedSchema.setColumns(relationalTableRegistrationServiceTestHelper.getExpectedSchemaColumns());
expectedSchema.setNullValue(EMPTY_STRING);
// Build an expected business object format.
BusinessObjectFormat expectedBusinessObjectFormat = new BusinessObjectFormat();
expectedBusinessObjectFormat.setId(businessObjectFormat.getId());
expectedBusinessObjectFormat.setNamespace(BDEF_NAMESPACE);
expectedBusinessObjectFormat.setBusinessObjectDefinitionName(BDEF_NAME);
expectedBusinessObjectFormat.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
expectedBusinessObjectFormat.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
expectedBusinessObjectFormat.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
expectedBusinessObjectFormat.setLatestVersion(LATEST_VERSION_FLAG_SET);
expectedBusinessObjectFormat.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
expectedBusinessObjectFormat.setBusinessObjectFormatParents(new ArrayList<>());
expectedBusinessObjectFormat.setBusinessObjectFormatChildren(new ArrayList<>());
expectedBusinessObjectFormat.setAttributeDefinitions(new ArrayList<>());
expectedBusinessObjectFormat.setAttributes(Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_SCHEMA_NAME), relationalSchemaName), new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_TABLE_NAME), relationalTableName)));
expectedBusinessObjectFormat.setSchema(expectedSchema);
// Validate the newly created business object format.
assertEquals(expectedBusinessObjectFormat, businessObjectFormat);
// Create a business object definition key.
BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(BDEF_NAMESPACE, BDEF_NAME);
// Retrieve business object definition that was created as part of the relational table registration.
BusinessObjectDefinitionEntity businessObjectDefinitionEntity = businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(businessObjectDefinitionKey);
// Validate the newly created business object definition.
assertEquals(existingBusinessObjectDefinitionEntity, businessObjectDefinitionEntity);
}
use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.
the class StorageUnitHelper method createStorageUnitsFromEntities.
/**
* Creates a list of storage units from the list of storage unit entities.
*
* @param storageUnitEntities the storage unit entities.
* @param includeStorageUnitStatusHistory specifies to include storage unit status history for each storage unit in the response
*
* @return the list of storage units.
*/
public List<StorageUnit> createStorageUnitsFromEntities(Collection<StorageUnitEntity> storageUnitEntities, Boolean includeStorageUnitStatusHistory) {
List<StorageUnit> storageUnits = new ArrayList<>();
for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
StorageUnit storageUnit = new StorageUnit();
storageUnits.add(storageUnit);
Storage storage = new Storage();
storageUnit.setStorage(storage);
StorageEntity storageEntity = storageUnitEntity.getStorage();
storage.setName(storageEntity.getName());
storage.setStoragePlatformName(storageEntity.getStoragePlatform().getName());
// Add the storage attributes.
if (!CollectionUtils.isEmpty(storageEntity.getAttributes())) {
List<Attribute> storageAttributes = new ArrayList<>();
storage.setAttributes(storageAttributes);
for (StorageAttributeEntity storageAttributeEntity : storageEntity.getAttributes()) {
Attribute attribute = new Attribute();
storageAttributes.add(attribute);
attribute.setName(storageAttributeEntity.getName());
attribute.setValue(storageAttributeEntity.getValue());
}
}
// Add the storage directory.
if (storageUnitEntity.getDirectoryPath() != null) {
StorageDirectory storageDirectory = new StorageDirectory();
storageUnit.setStorageDirectory(storageDirectory);
storageDirectory.setDirectoryPath(storageUnitEntity.getDirectoryPath());
}
// Add the storage files.
if (!storageUnitEntity.getStorageFiles().isEmpty()) {
List<StorageFile> storageFiles = new ArrayList<>();
storageUnit.setStorageFiles(storageFiles);
for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
storageFiles.add(storageFileHelper.createStorageFileFromEntity(storageFileEntity));
}
}
// Set the storage unit status.
storageUnit.setStorageUnitStatus(storageUnitEntity.getStatus().getCode());
// If specified, add storage unit status history.
if (BooleanUtils.isTrue(includeStorageUnitStatusHistory)) {
List<StorageUnitStatusChangeEvent> storageUnitStatusChangeEvents = new ArrayList<>();
storageUnit.setStorageUnitStatusHistory(storageUnitStatusChangeEvents);
for (StorageUnitStatusHistoryEntity storageUnitStatusHistoryEntity : storageUnitEntity.getHistoricalStatuses()) {
storageUnitStatusChangeEvents.add(new StorageUnitStatusChangeEvent(storageUnitStatusHistoryEntity.getStatus().getCode(), HerdDateUtils.getXMLGregorianCalendarValue(storageUnitStatusHistoryEntity.getCreatedOn()), storageUnitStatusHistoryEntity.getCreatedBy()));
}
}
// Set the number of failed attempts to execute a storage policy transition.
storageUnit.setStoragePolicyTransitionFailedAttempts(storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
if (storageUnitEntity.getRestoreExpirationOn() != null) {
storageUnit.setRestoreExpirationOn(HerdDateUtils.getXMLGregorianCalendarValue(storageUnitEntity.getRestoreExpirationOn()));
}
}
return storageUnits;
}
use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.
the class GetBusinessObjectDataTest method testGetBusinessObjectDataIncludeStorageUnitStatusHistory.
@Test
public void testGetBusinessObjectDataIncludeStorageUnitStatusHistory() throws Exception {
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
// Create a business object data storage unit key.
BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey = new BusinessObjectDataStorageUnitKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME);
// Create a storage unit entity.
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, StorageUnitStatusEntity.DISABLED);
// Execute a storage unit status change.
businessObjectDataStorageUnitStatusService.updateBusinessObjectDataStorageUnitStatus(businessObjectDataStorageUnitKey, new BusinessObjectDataStorageUnitStatusUpdateRequest(StorageUnitStatusEntity.ENABLED));
List<FieldExtension> fieldExtensionList = new ArrayList<>();
fieldExtensionList.add(buildFieldExtension("namespace", "${namespace}"));
fieldExtensionList.add(buildFieldExtension("businessObjectDefinitionName", "${businessObjectDefinitionName}"));
fieldExtensionList.add(buildFieldExtension("businessObjectFormatUsage", "${businessObjectFormatUsage}"));
fieldExtensionList.add(buildFieldExtension("businessObjectFormatFileType", "${businessObjectFormatFileType}"));
fieldExtensionList.add(buildFieldExtension("partitionValue", "${partitionValue}"));
fieldExtensionList.add(buildFieldExtension("subPartitionValues", "${subPartitionValues}"));
fieldExtensionList.add(buildFieldExtension("businessObjectFormatVersion", "${businessObjectFormatVersion}"));
fieldExtensionList.add(buildFieldExtension("businessObjectDataVersion", "${businessObjectDataVersion}"));
fieldExtensionList.add(buildFieldExtension("businessObjectDataStatus", "${businessObjectDataStatus}"));
fieldExtensionList.add(buildFieldExtension("includeBusinessObjectDataStatusHistory", "${includeBusinessObjectDataStatusHistory}"));
fieldExtensionList.add(buildFieldExtension("includeStorageUnitStatusHistory", "${includeStorageUnitStatusHistory}"));
List<Parameter> parameters = new ArrayList<>();
parameters.add(buildParameter("namespace", BDEF_NAMESPACE));
parameters.add(buildParameter("businessObjectDefinitionName", BDEF_NAME));
parameters.add(buildParameter("businessObjectFormatUsage", FORMAT_USAGE_CODE));
parameters.add(buildParameter("businessObjectFormatFileType", FORMAT_FILE_TYPE_CODE));
parameters.add(buildParameter("partitionValue", PARTITION_VALUE));
parameters.add(buildParameter("subPartitionValues", herdStringHelper.buildStringWithDefaultDelimiter(SUBPARTITION_VALUES)));
parameters.add(buildParameter("businessObjectFormatVersion", FORMAT_VERSION.toString()));
parameters.add(buildParameter("businessObjectDataVersion", DATA_VERSION.toString()));
parameters.add(buildParameter("businessObjectDataStatus", BusinessObjectDataStatusEntity.VALID));
parameters.add(buildParameter("includeBusinessObjectDataStatusHistory", NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY.toString()));
parameters.add(buildParameter("includeStorageUnitStatusHistory", INCLUDE_STORAGE_UNIT_STATUS_HISTORY.toString()));
// Build the expected response object. The storage unit history record is expected to have system username for the createdBy auditable field.
BusinessObjectData expectedBusinessObjectData = new BusinessObjectData(storageUnitEntity.getBusinessObjectData().getId(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, Arrays.asList(new StorageUnit(new Storage(STORAGE_NAME, StoragePlatformEntity.S3, null), NO_STORAGE_DIRECTORY, null, StorageUnitStatusEntity.ENABLED, Arrays.asList(new StorageUnitStatusChangeEvent(StorageUnitStatusEntity.ENABLED, HerdDateUtils.getXMLGregorianCalendarValue(IterableUtils.get(storageUnitEntity.getHistoricalStatuses(), 0).getCreatedOn()), HerdDaoSecurityHelper.SYSTEM_USER)), NO_STORAGE_POLICY_TRANSITION_FAILED_ATTEMPTS, NO_RESTORE_EXPIRATION_ON)), NO_ATTRIBUTES, NO_BUSINESS_OBJECT_DATA_PARENTS, NO_BUSINESS_OBJECT_DATA_CHILDREN, NO_BUSINESS_OBJECT_DATA_STATUS_HISTORY);
// Run the activiti task and validate the returned response object.
Map<String, Object> variableValuesToValidate = new HashMap<>();
variableValuesToValidate.put(GetBusinessObjectDataStatus.VARIABLE_JSON_RESPONSE, jsonHelper.objectToJson(expectedBusinessObjectData));
testActivitiServiceTaskSuccess(GetBusinessObjectData.class.getCanonicalName(), fieldExtensionList, parameters, variableValuesToValidate);
}
use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.
the class UploaderController method performUpload.
/**
* Executes the uploader workflow.
*
* @param regServerAccessParamsDto the DTO for the parameters required to communicate with the registration server
* @param manifestPath the local path to the manifest file
* @param params the S3 file transfer request parameters being used to pass the following arguments: <ul> <li><code>s3AccessKey</code> the S3 access key
* <li><code>s3SecretKey</code> the S3 secret key <li><code>localPath</code> the local path to directory containing data files
* <li><code>httpProxyHost</code> the HTTP proxy host <li><code>httpProxyPort</code> the HTTP proxy port <li><code>maxThreads</code> the maximum number of
* threads to use for file transfer to S3< <li><code>useRrs</code> specifies whether S3 reduced redundancy storage option will be used when copying to S3
* </ul>
* @param createNewVersion if not set, only initial version of the business object data is allowed to be created
* @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
* @param maxRetryAttempts the maximum number of the business object data registration retry attempts
* @param retryDelaySecs the delay in seconds between the business object data registration retry attempts
*
* @throws InterruptedException if the upload thread was interrupted.
* @throws JAXBException if a JAXB error was encountered.
* @throws IOException if an I/O error was encountered.
* @throws URISyntaxException if a URI syntax error was encountered.
*/
@SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.")
public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params, Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs) throws InterruptedException, JAXBException, IOException, URISyntaxException {
boolean cleanUpS3KeyPrefixOnFailure = false;
BusinessObjectDataKey businessObjectDataKey = null;
try {
// Process manifest file
UploaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
String storageName = getStorageNameFromManifest(manifest);
manifest.setStorageName(storageName);
// Validate local files and prepare a list of source files to copy to S3.
List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles());
// Validate that we do not have duplicate files listed in the manifest file.
List<File> duplicateFiles = findDuplicateFiles(sourceFiles);
if (!duplicateFiles.isEmpty()) {
throw new IllegalArgumentException(String.format("Manifest contains duplicate file names. Duplicates: [\"%s\"]", StringUtils.join(duplicateFiles, "\", \"")));
}
// Initialize uploader web client.
uploaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
// Handle the latest business object data version if one exists.
checkLatestBusinessObjectDataVersion(manifest, force);
// Pre-register a new version of business object data in UPLOADING state with the registration server.
BusinessObjectData businessObjectData = uploaderWebClient.preRegisterBusinessObjectData(manifest, storageName, createNewVersion);
// Get business object data key.
businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
// Get the business object data version.
Integer businessObjectDataVersion = businessObjectDataKey.getBusinessObjectDataVersion();
// Add credential provider.
params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() {
@Override
public AwsCredential getNewAwsCredential() throws Exception {
return uploaderWebClient.getBusinessObjectDataUploadCredential(manifest, storageName, businessObjectDataVersion, null).getAwsCredential();
}
});
// Get S3 key prefix from the business object data pre-registration response.
String s3KeyPrefix = IterableUtils.get(businessObjectData.getStorageUnits(), 0).getStorageDirectory().getDirectoryPath();
// Get S3 bucket information.
Storage storage = uploaderWebClient.getStorage(storageName);
// Get S3 bucket name. Please note that since this value is required we pass a "true" flag.
String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
// Set the KMS ID, if available
String kmsKeyId = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storage, false);
params.setKmsKeyId(kmsKeyId);
// Special handling for the maxThreads command line option.
params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS));
// Populate several missing fields in the S3 file transfer request parameters DTO.
params.setS3BucketName(s3BucketName);
// Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
params.setS3KeyPrefix(s3KeyPrefix + "/");
params.setFiles(sourceFiles);
// When listing S3 files, by default, we do not ignore 0 byte objects that represent S3 directories.
if (s3Service.listDirectory(params).isEmpty()) {
cleanUpS3KeyPrefixOnFailure = true;
} else {
throw new IllegalStateException(String.format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(), params.getS3KeyPrefix()));
}
// Upload files.
s3Service.uploadFileList(params);
// Get the list of files uploaded to S3 key prefix.
if (LOGGER.isInfoEnabled()) {
logS3KeyPrefixContents(params);
}
// Add storage files to the business object data.
addStorageFilesWithRetry(businessObjectDataKey, manifest, params, storage.getName(), maxRetryAttempts, retryDelaySecs);
// Change status of the business object data to VALID.
uploaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
} catch (InterruptedException | JAXBException | IOException | URISyntaxException e) {
// occurred, let's rollback the data transfer (clean up the S3 key prefix).
if (cleanUpS3KeyPrefixOnFailure) {
LOGGER.info(String.format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(), params.getS3BucketName()));
s3Service.deleteDirectoryIgnoreException(params);
}
// If a new business object data version got pre-registered, update it's status to INVALID.
if (businessObjectDataKey != null) {
uploaderWebClient.updateBusinessObjectDataStatusIgnoreException(businessObjectDataKey, BusinessObjectDataStatusEntity.INVALID);
}
throw e;
}
}
use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.
the class DataBridgeWebClientTest method testGetStorage.
/**
* Calls getStorage() method and makes assertions.
*
* @param useSsl specifies whether to use SSL or not
*
* @throws IOException
* @throws JAXBException
* @throws URISyntaxException
*/
private void testGetStorage(boolean useSsl) throws IOException, JAXBException, URISyntaxException {
dataBridgeWebClient.regServerAccessParamsDto.setUseSsl(useSsl);
String expectedStorageName = "testStorage";
Storage storage = dataBridgeWebClient.getStorage(expectedStorageName);
assertNotNull("storage is null", storage);
assertEquals("storage name", expectedStorageName, storage.getName());
}
Aggregations