Search in sources :

Example 21 with Storage

use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.

the class RelationalTableRegistrationServiceTest method testCreateRelationalTableRegistrationWithAppendToExistingBusinessObjectDefinitionSetToTrue.

@Test
public void testCreateRelationalTableRegistrationWithAppendToExistingBusinessObjectDefinitionSetToTrue() {
    // Create an existing business object definition.
    BusinessObjectDefinitionEntity existingBusinessObjectDefinitionEntity = businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(BDEF_NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, DESCRIPTION, BDEF_DISPLAY_NAME, new ArrayList<>());
    // Create database entities required for relational table registration testing.
    relationalTableRegistrationServiceTestHelper.createDatabaseEntitiesForRelationalTableRegistrationTesting(STORAGE_NAME);
    // Pick one of the in-memory database tables to be registered as a relational table.
    String relationalSchemaName = "PUBLIC";
    String relationalTableName = BusinessObjectDefinitionEntity.TABLE_NAME.toUpperCase();
    // Create a relational table registration create request for a table that is part of the in-memory database setup as part of DAO mocks.
    RelationalTableRegistrationCreateRequest relationalTableRegistrationCreateRequest = new RelationalTableRegistrationCreateRequest(BDEF_NAMESPACE, BDEF_NAME, BDEF_DISPLAY_NAME, FORMAT_USAGE_CODE, DATA_PROVIDER_NAME, relationalSchemaName, relationalTableName, STORAGE_NAME);
    // Create a relational table registration.
    BusinessObjectData businessObjectData = relationalTableRegistrationService.createRelationalTableRegistration(relationalTableRegistrationCreateRequest, APPEND_TO_EXISTING_BUSINESS_OBJECT_DEFINTION_TRUE);
    // Create an expected storage unit.
    StorageUnit expectedStorageUnit = new StorageUnit();
    expectedStorageUnit.setStorage(new Storage(STORAGE_NAME, StoragePlatformEntity.RELATIONAL, relationalTableRegistrationServiceTestHelper.getStorageAttributes()));
    expectedStorageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
    // Create an expected business object data.
    BusinessObjectData expectedBusinessObjectData = new BusinessObjectData();
    expectedBusinessObjectData.setId(businessObjectData.getId());
    expectedBusinessObjectData.setNamespace(BDEF_NAMESPACE);
    expectedBusinessObjectData.setBusinessObjectDefinitionName(BDEF_NAME);
    expectedBusinessObjectData.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    expectedBusinessObjectData.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
    expectedBusinessObjectData.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
    expectedBusinessObjectData.setPartitionValue(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_VALUE);
    expectedBusinessObjectData.setSubPartitionValues(new ArrayList<>());
    expectedBusinessObjectData.setVersion(INITIAL_DATA_VERSION);
    expectedBusinessObjectData.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
    expectedBusinessObjectData.setLatestVersion(LATEST_VERSION_FLAG_SET);
    expectedBusinessObjectData.setStatus(BusinessObjectDataStatusEntity.VALID);
    expectedBusinessObjectData.setStorageUnits(Collections.singletonList(expectedStorageUnit));
    expectedBusinessObjectData.setAttributes(new ArrayList<>());
    expectedBusinessObjectData.setBusinessObjectDataParents(new ArrayList<>());
    expectedBusinessObjectData.setBusinessObjectDataChildren(new ArrayList<>());
    // Validate the response.
    assertEquals(expectedBusinessObjectData, businessObjectData);
    // Create a business object format key.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE, INITIAL_FORMAT_VERSION);
    // Retrieve business object format that was created as part of the relational table registration.
    BusinessObjectFormat businessObjectFormat = businessObjectFormatService.getBusinessObjectFormat(businessObjectFormatKey);
    // Create an expected schema.
    Schema expectedSchema = new Schema();
    expectedSchema.setColumns(relationalTableRegistrationServiceTestHelper.getExpectedSchemaColumns());
    expectedSchema.setNullValue(EMPTY_STRING);
    // Build an expected business object format.
    BusinessObjectFormat expectedBusinessObjectFormat = new BusinessObjectFormat();
    expectedBusinessObjectFormat.setId(businessObjectFormat.getId());
    expectedBusinessObjectFormat.setNamespace(BDEF_NAMESPACE);
    expectedBusinessObjectFormat.setBusinessObjectDefinitionName(BDEF_NAME);
    expectedBusinessObjectFormat.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    expectedBusinessObjectFormat.setBusinessObjectFormatFileType(FileTypeEntity.RELATIONAL_TABLE_FILE_TYPE);
    expectedBusinessObjectFormat.setBusinessObjectFormatVersion(INITIAL_FORMAT_VERSION);
    expectedBusinessObjectFormat.setLatestVersion(LATEST_VERSION_FLAG_SET);
    expectedBusinessObjectFormat.setPartitionKey(BusinessObjectDataServiceImpl.NO_PARTITIONING_PARTITION_KEY);
    expectedBusinessObjectFormat.setBusinessObjectFormatParents(new ArrayList<>());
    expectedBusinessObjectFormat.setBusinessObjectFormatChildren(new ArrayList<>());
    expectedBusinessObjectFormat.setAttributeDefinitions(new ArrayList<>());
    expectedBusinessObjectFormat.setAttributes(Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_SCHEMA_NAME), relationalSchemaName), new Attribute(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_FORMAT_ATTRIBUTE_NAME_RELATIONAL_TABLE_NAME), relationalTableName)));
    expectedBusinessObjectFormat.setSchema(expectedSchema);
    // Validate the newly created business object format.
    assertEquals(expectedBusinessObjectFormat, businessObjectFormat);
    // Create a business object definition key.
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(BDEF_NAMESPACE, BDEF_NAME);
    // Retrieve business object definition that was created as part of the relational table registration.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity = businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(businessObjectDefinitionKey);
    // Validate the newly created business object definition.
    assertEquals(existingBusinessObjectDefinitionEntity, businessObjectDefinitionEntity);
}
Also used : Storage(org.finra.herd.model.api.xml.Storage) RelationalTableRegistrationCreateRequest(org.finra.herd.model.api.xml.RelationalTableRegistrationCreateRequest) BusinessObjectData(org.finra.herd.model.api.xml.BusinessObjectData) Attribute(org.finra.herd.model.api.xml.Attribute) BusinessObjectDefinitionKey(org.finra.herd.model.api.xml.BusinessObjectDefinitionKey) BusinessObjectDefinitionEntity(org.finra.herd.model.jpa.BusinessObjectDefinitionEntity) Schema(org.finra.herd.model.api.xml.Schema) BusinessObjectFormatKey(org.finra.herd.model.api.xml.BusinessObjectFormatKey) StorageUnit(org.finra.herd.model.api.xml.StorageUnit) BusinessObjectFormat(org.finra.herd.model.api.xml.BusinessObjectFormat) Test(org.junit.Test)

Example 22 with Storage

use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.

the class StorageUnitHelper method createStorageUnitsFromEntities.

/**
 * Creates a list of storage units from the list of storage unit entities.
 *
 * @param storageUnitEntities the storage unit entities.
 * @param includeStorageUnitStatusHistory specifies to include storage unit status history for each storage unit in the response
 *
 * @return the list of storage units.
 */
public List<StorageUnit> createStorageUnitsFromEntities(Collection<StorageUnitEntity> storageUnitEntities, Boolean includeStorageUnitStatusHistory) {
    List<StorageUnit> storageUnits = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        StorageUnit storageUnit = new StorageUnit();
        storageUnits.add(storageUnit);
        Storage storage = new Storage();
        storageUnit.setStorage(storage);
        StorageEntity storageEntity = storageUnitEntity.getStorage();
        storage.setName(storageEntity.getName());
        storage.setStoragePlatformName(storageEntity.getStoragePlatform().getName());
        // Add the storage attributes.
        if (!CollectionUtils.isEmpty(storageEntity.getAttributes())) {
            List<Attribute> storageAttributes = new ArrayList<>();
            storage.setAttributes(storageAttributes);
            for (StorageAttributeEntity storageAttributeEntity : storageEntity.getAttributes()) {
                Attribute attribute = new Attribute();
                storageAttributes.add(attribute);
                attribute.setName(storageAttributeEntity.getName());
                attribute.setValue(storageAttributeEntity.getValue());
            }
        }
        // Add the storage directory.
        if (storageUnitEntity.getDirectoryPath() != null) {
            StorageDirectory storageDirectory = new StorageDirectory();
            storageUnit.setStorageDirectory(storageDirectory);
            storageDirectory.setDirectoryPath(storageUnitEntity.getDirectoryPath());
        }
        // Add the storage files.
        if (!storageUnitEntity.getStorageFiles().isEmpty()) {
            List<StorageFile> storageFiles = new ArrayList<>();
            storageUnit.setStorageFiles(storageFiles);
            for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
                storageFiles.add(storageFileHelper.createStorageFileFromEntity(storageFileEntity));
            }
        }
        // Set the storage unit status.
        storageUnit.setStorageUnitStatus(storageUnitEntity.getStatus().getCode());
        // If specified, add storage unit status history.
        if (BooleanUtils.isTrue(includeStorageUnitStatusHistory)) {
            List<StorageUnitStatusChangeEvent> storageUnitStatusChangeEvents = new ArrayList<>();
            storageUnit.setStorageUnitStatusHistory(storageUnitStatusChangeEvents);
            for (StorageUnitStatusHistoryEntity storageUnitStatusHistoryEntity : storageUnitEntity.getHistoricalStatuses()) {
                storageUnitStatusChangeEvents.add(new StorageUnitStatusChangeEvent(storageUnitStatusHistoryEntity.getStatus().getCode(), HerdDateUtils.getXMLGregorianCalendarValue(storageUnitStatusHistoryEntity.getCreatedOn()), storageUnitStatusHistoryEntity.getCreatedBy()));
            }
        }
        // Set the number of failed attempts to execute a storage policy transition.
        storageUnit.setStoragePolicyTransitionFailedAttempts(storageUnitEntity.getStoragePolicyTransitionFailedAttempts());
        if (storageUnitEntity.getRestoreExpirationOn() != null) {
            storageUnit.setRestoreExpirationOn(HerdDateUtils.getXMLGregorianCalendarValue(storageUnitEntity.getRestoreExpirationOn()));
        }
    }
    return storageUnits;
}
Also used : StorageUnitStatusChangeEvent(org.finra.herd.model.api.xml.StorageUnitStatusChangeEvent) StorageUnitStatusHistoryEntity(org.finra.herd.model.jpa.StorageUnitStatusHistoryEntity) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) Attribute(org.finra.herd.model.api.xml.Attribute) StorageAttributeEntity(org.finra.herd.model.jpa.StorageAttributeEntity) ArrayList(java.util.ArrayList) StorageUnit(org.finra.herd.model.api.xml.StorageUnit) StorageEntity(org.finra.herd.model.jpa.StorageEntity) StorageDirectory(org.finra.herd.model.api.xml.StorageDirectory) Storage(org.finra.herd.model.api.xml.Storage) StorageFileEntity(org.finra.herd.model.jpa.StorageFileEntity) StorageFile(org.finra.herd.model.api.xml.StorageFile)

Example 23 with Storage

use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.

the class GetBusinessObjectDataTest method testGetBusinessObjectDataIncludeStorageUnitStatusHistory.

@Test
public void testGetBusinessObjectDataIncludeStorageUnitStatusHistory() throws Exception {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a business object data storage unit key.
    BusinessObjectDataStorageUnitKey businessObjectDataStorageUnitKey = new BusinessObjectDataStorageUnitKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME);
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKey, StorageUnitStatusEntity.DISABLED);
    // Execute a storage unit status change.
    businessObjectDataStorageUnitStatusService.updateBusinessObjectDataStorageUnitStatus(businessObjectDataStorageUnitKey, new BusinessObjectDataStorageUnitStatusUpdateRequest(StorageUnitStatusEntity.ENABLED));
    List<FieldExtension> fieldExtensionList = new ArrayList<>();
    fieldExtensionList.add(buildFieldExtension("namespace", "${namespace}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectDefinitionName", "${businessObjectDefinitionName}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectFormatUsage", "${businessObjectFormatUsage}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectFormatFileType", "${businessObjectFormatFileType}"));
    fieldExtensionList.add(buildFieldExtension("partitionValue", "${partitionValue}"));
    fieldExtensionList.add(buildFieldExtension("subPartitionValues", "${subPartitionValues}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectFormatVersion", "${businessObjectFormatVersion}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectDataVersion", "${businessObjectDataVersion}"));
    fieldExtensionList.add(buildFieldExtension("businessObjectDataStatus", "${businessObjectDataStatus}"));
    fieldExtensionList.add(buildFieldExtension("includeBusinessObjectDataStatusHistory", "${includeBusinessObjectDataStatusHistory}"));
    fieldExtensionList.add(buildFieldExtension("includeStorageUnitStatusHistory", "${includeStorageUnitStatusHistory}"));
    List<Parameter> parameters = new ArrayList<>();
    parameters.add(buildParameter("namespace", BDEF_NAMESPACE));
    parameters.add(buildParameter("businessObjectDefinitionName", BDEF_NAME));
    parameters.add(buildParameter("businessObjectFormatUsage", FORMAT_USAGE_CODE));
    parameters.add(buildParameter("businessObjectFormatFileType", FORMAT_FILE_TYPE_CODE));
    parameters.add(buildParameter("partitionValue", PARTITION_VALUE));
    parameters.add(buildParameter("subPartitionValues", herdStringHelper.buildStringWithDefaultDelimiter(SUBPARTITION_VALUES)));
    parameters.add(buildParameter("businessObjectFormatVersion", FORMAT_VERSION.toString()));
    parameters.add(buildParameter("businessObjectDataVersion", DATA_VERSION.toString()));
    parameters.add(buildParameter("businessObjectDataStatus", BusinessObjectDataStatusEntity.VALID));
    parameters.add(buildParameter("includeBusinessObjectDataStatusHistory", NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY.toString()));
    parameters.add(buildParameter("includeStorageUnitStatusHistory", INCLUDE_STORAGE_UNIT_STATUS_HISTORY.toString()));
    // Build the expected response object. The storage unit history record is expected to have system username for the createdBy auditable field.
    BusinessObjectData expectedBusinessObjectData = new BusinessObjectData(storageUnitEntity.getBusinessObjectData().getId(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, Arrays.asList(new StorageUnit(new Storage(STORAGE_NAME, StoragePlatformEntity.S3, null), NO_STORAGE_DIRECTORY, null, StorageUnitStatusEntity.ENABLED, Arrays.asList(new StorageUnitStatusChangeEvent(StorageUnitStatusEntity.ENABLED, HerdDateUtils.getXMLGregorianCalendarValue(IterableUtils.get(storageUnitEntity.getHistoricalStatuses(), 0).getCreatedOn()), HerdDaoSecurityHelper.SYSTEM_USER)), NO_STORAGE_POLICY_TRANSITION_FAILED_ATTEMPTS, NO_RESTORE_EXPIRATION_ON)), NO_ATTRIBUTES, NO_BUSINESS_OBJECT_DATA_PARENTS, NO_BUSINESS_OBJECT_DATA_CHILDREN, NO_BUSINESS_OBJECT_DATA_STATUS_HISTORY);
    // Run the activiti task and validate the returned response object.
    Map<String, Object> variableValuesToValidate = new HashMap<>();
    variableValuesToValidate.put(GetBusinessObjectDataStatus.VARIABLE_JSON_RESPONSE, jsonHelper.objectToJson(expectedBusinessObjectData));
    testActivitiServiceTaskSuccess(GetBusinessObjectData.class.getCanonicalName(), fieldExtensionList, parameters, variableValuesToValidate);
}
Also used : StorageUnitStatusChangeEvent(org.finra.herd.model.api.xml.StorageUnitStatusChangeEvent) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) BusinessObjectData(org.finra.herd.model.api.xml.BusinessObjectData) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) StorageUnit(org.finra.herd.model.api.xml.StorageUnit) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) BusinessObjectDataStorageUnitKey(org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey) Storage(org.finra.herd.model.api.xml.Storage) FieldExtension(org.activiti.bpmn.model.FieldExtension) Parameter(org.finra.herd.model.api.xml.Parameter) BusinessObjectDataStorageUnitStatusUpdateRequest(org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitStatusUpdateRequest) Test(org.junit.Test)

Example 24 with Storage

use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.

the class UploaderController method performUpload.

/**
 * Executes the uploader workflow.
 *
 * @param regServerAccessParamsDto the DTO for the parameters required to communicate with the registration server
 * @param manifestPath the local path to the manifest file
 * @param params the S3 file transfer request parameters being used to pass the following arguments: <ul> <li><code>s3AccessKey</code> the S3 access key
 * <li><code>s3SecretKey</code> the S3 secret key <li><code>localPath</code> the local path to directory containing data files
 * <li><code>httpProxyHost</code> the HTTP proxy host <li><code>httpProxyPort</code> the HTTP proxy port <li><code>maxThreads</code> the maximum number of
 * threads to use for file transfer to S3< <li><code>useRrs</code> specifies whether S3 reduced redundancy storage option will be used when copying to S3
 * </ul>
 * @param createNewVersion if not set, only initial version of the business object data is allowed to be created
 * @param force if set, allows upload to proceed when the latest version of the business object data has UPLOADING status by invalidating that version
 * @param maxRetryAttempts the maximum number of the business object data registration retry attempts
 * @param retryDelaySecs the delay in seconds between the business object data registration retry attempts
 *
 * @throws InterruptedException if the upload thread was interrupted.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws IOException if an I/O error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.")
public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params, Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs) throws InterruptedException, JAXBException, IOException, URISyntaxException {
    boolean cleanUpS3KeyPrefixOnFailure = false;
    BusinessObjectDataKey businessObjectDataKey = null;
    try {
        // Process manifest file
        UploaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
        String storageName = getStorageNameFromManifest(manifest);
        manifest.setStorageName(storageName);
        // Validate local files and prepare a list of source files to copy to S3.
        List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles());
        // Validate that we do not have duplicate files listed in the manifest file.
        List<File> duplicateFiles = findDuplicateFiles(sourceFiles);
        if (!duplicateFiles.isEmpty()) {
            throw new IllegalArgumentException(String.format("Manifest contains duplicate file names. Duplicates: [\"%s\"]", StringUtils.join(duplicateFiles, "\", \"")));
        }
        // Initialize uploader web client.
        uploaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
        // Handle the latest business object data version if one exists.
        checkLatestBusinessObjectDataVersion(manifest, force);
        // Pre-register a new version of business object data in UPLOADING state with the registration server.
        BusinessObjectData businessObjectData = uploaderWebClient.preRegisterBusinessObjectData(manifest, storageName, createNewVersion);
        // Get business object data key.
        businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
        // Get the business object data version.
        Integer businessObjectDataVersion = businessObjectDataKey.getBusinessObjectDataVersion();
        // Add credential provider.
        params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() {

            @Override
            public AwsCredential getNewAwsCredential() throws Exception {
                return uploaderWebClient.getBusinessObjectDataUploadCredential(manifest, storageName, businessObjectDataVersion, null).getAwsCredential();
            }
        });
        // Get S3 key prefix from the business object data pre-registration response.
        String s3KeyPrefix = IterableUtils.get(businessObjectData.getStorageUnits(), 0).getStorageDirectory().getDirectoryPath();
        // Get S3 bucket information.
        Storage storage = uploaderWebClient.getStorage(storageName);
        // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
        String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
        // Set the KMS ID, if available
        String kmsKeyId = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storage, false);
        params.setKmsKeyId(kmsKeyId);
        // Special handling for the maxThreads command line option.
        params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS));
        // Populate several missing fields in the S3 file transfer request parameters DTO.
        params.setS3BucketName(s3BucketName);
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        params.setS3KeyPrefix(s3KeyPrefix + "/");
        params.setFiles(sourceFiles);
        // When listing S3 files, by default, we do not ignore 0 byte objects that represent S3 directories.
        if (s3Service.listDirectory(params).isEmpty()) {
            cleanUpS3KeyPrefixOnFailure = true;
        } else {
            throw new IllegalStateException(String.format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(), params.getS3KeyPrefix()));
        }
        // Upload files.
        s3Service.uploadFileList(params);
        // Get the list of files uploaded to S3 key prefix.
        if (LOGGER.isInfoEnabled()) {
            logS3KeyPrefixContents(params);
        }
        // Add storage files to the business object data.
        addStorageFilesWithRetry(businessObjectDataKey, manifest, params, storage.getName(), maxRetryAttempts, retryDelaySecs);
        // Change status of the business object data to VALID.
        uploaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
    } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) {
        // occurred, let's rollback the data transfer (clean up the S3 key prefix).
        if (cleanUpS3KeyPrefixOnFailure) {
            LOGGER.info(String.format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(), params.getS3BucketName()));
            s3Service.deleteDirectoryIgnoreException(params);
        }
        // If a new business object data version got pre-registered, update it's status to INVALID.
        if (businessObjectDataKey != null) {
            uploaderWebClient.updateBusinessObjectDataStatusIgnoreException(businessObjectDataKey, BusinessObjectDataStatusEntity.INVALID);
        }
        throw e;
    }
}
Also used : UploaderInputManifestDto(org.finra.herd.model.dto.UploaderInputManifestDto) BusinessObjectData(org.finra.herd.model.api.xml.BusinessObjectData) AutoRefreshCredentialProvider(org.finra.herd.tools.common.databridge.AutoRefreshCredentialProvider) JAXBException(javax.xml.bind.JAXBException) IOException(java.io.IOException) URISyntaxException(java.net.URISyntaxException) BusinessObjectDataKey(org.finra.herd.model.api.xml.BusinessObjectDataKey) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) JAXBException(javax.xml.bind.JAXBException) Storage(org.finra.herd.model.api.xml.Storage) ManifestFile(org.finra.herd.model.dto.ManifestFile) File(java.io.File) AwsCredential(org.finra.herd.model.api.xml.AwsCredential) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings)

Example 25 with Storage

use of org.finra.herd.model.api.xml.Storage in project herd by FINRAOS.

the class DataBridgeWebClientTest method testGetStorage.

/**
 * Calls getStorage() method and makes assertions.
 *
 * @param useSsl specifies whether to use SSL or not
 *
 * @throws IOException
 * @throws JAXBException
 * @throws URISyntaxException
 */
private void testGetStorage(boolean useSsl) throws IOException, JAXBException, URISyntaxException {
    dataBridgeWebClient.regServerAccessParamsDto.setUseSsl(useSsl);
    String expectedStorageName = "testStorage";
    Storage storage = dataBridgeWebClient.getStorage(expectedStorageName);
    assertNotNull("storage is null", storage);
    assertEquals("storage name", expectedStorageName, storage.getName());
}
Also used : Storage(org.finra.herd.model.api.xml.Storage)

Aggregations

Storage (org.finra.herd.model.api.xml.Storage)38 Test (org.junit.Test)29 StorageUnit (org.finra.herd.model.api.xml.StorageUnit)19 BusinessObjectData (org.finra.herd.model.api.xml.BusinessObjectData)17 Attribute (org.finra.herd.model.api.xml.Attribute)10 StorageKey (org.finra.herd.model.api.xml.StorageKey)10 ArrayList (java.util.ArrayList)6 StorageCreateRequest (org.finra.herd.model.api.xml.StorageCreateRequest)6 File (java.io.File)5 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)4 S3KeyPrefixInformation (org.finra.herd.model.api.xml.S3KeyPrefixInformation)4 StorageAttributesUpdateRequest (org.finra.herd.model.api.xml.StorageAttributesUpdateRequest)4 StorageDirectory (org.finra.herd.model.api.xml.StorageDirectory)4 DownloaderInputManifestDto (org.finra.herd.model.dto.DownloaderInputManifestDto)4 DownloaderOutputManifestDto (org.finra.herd.model.dto.DownloaderOutputManifestDto)4 StorageUnitEntity (org.finra.herd.model.jpa.StorageUnitEntity)4 IOException (java.io.IOException)3 Path (java.nio.file.Path)3 AwsCredential (org.finra.herd.model.api.xml.AwsCredential)3 BusinessObjectDataStorageUnitKey (org.finra.herd.model.api.xml.BusinessObjectDataStorageUnitKey)3