Examples with UploadSingleInitiationResponse org.finra.herd.model.api.xml.UploadSingleInitiationResponse used on opensource projects

Example 11 with UploadSingleInitiationResponse

use of org.finra.herd.model.api.xml.UploadSingleInitiationResponse in project herd by FINRAOS.

the class HerdJmsMessageListenerTest method testS3MessageS3FileNoExists.

@Test
public void testS3MessageS3FileNoExists() throws Exception {
    setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF);
    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();
    UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, TARGET_S3_KEY));
    String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null);
    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));
    S3EventNotification s3EventNotification = new S3EventNotification(records);
    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.OFF);
    // Try to process an S3 JMS message, when source S3 file does not exist.
    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}

Example 12 with UploadSingleInitiationResponse

use of org.finra.herd.model.api.xml.UploadSingleInitiationResponse in project herd by FINRAOS.

the class HerdJmsMessageListenerTest method testS3Message.

@Test
public void testS3Message() throws Exception {
    setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF);
    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();
    UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest());
    String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null);
    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));
    S3EventNotification s3EventNotification = new S3EventNotification(records);
    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.DEBUG);
    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}

Example 13 with UploadSingleInitiationResponse

use of org.finra.herd.model.api.xml.UploadSingleInitiationResponse in project herd by FINRAOS.

the class UploadDownloadServiceImpl method initiateUploadSingle.

@PublishNotificationMessages
@NamespacePermission(fields = { "#uploadSingleInitiationRequest?.sourceBusinessObjectFormatKey?.namespace", "#uploadSingleInitiationRequest?.targetBusinessObjectFormatKey?.namespace" }, permissions = NamespacePermissionEnum.WRITE)
@Override
public UploadSingleInitiationResponse initiateUploadSingle(UploadSingleInitiationRequest uploadSingleInitiationRequest) {
    // Validate and trim the request parameters.
    validateUploadSingleInitiationRequest(uploadSingleInitiationRequest);
    // Get the business object format for the specified parameters and make sure it exists.
    BusinessObjectFormatEntity sourceBusinessObjectFormatEntity = businessObjectFormatDaoHelper.getBusinessObjectFormatEntity(uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey());
    // Get the target business object format entity for the specified parameters and make sure it exists.
    BusinessObjectFormatEntity targetBusinessObjectFormatEntity = businessObjectFormatDaoHelper.getBusinessObjectFormatEntity(uploadSingleInitiationRequest.getTargetBusinessObjectFormatKey());
    // Get the S3 managed "loading dock" storage entity and make sure it exists.
    StorageEntity sourceStorageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    // Get S3 bucket name for the storage. Please note that since those values are required we pass a "true" flag.
    String s3BucketName = storageHelper.getStorageBucketName(sourceStorageEntity);
    // Get the S3 managed "external" storage entity and make sure it exists.
    String targetStorageName;
    if (uploadSingleInitiationRequest.getTargetStorageName() != null) {
        targetStorageName = uploadSingleInitiationRequest.getTargetStorageName();
    } else {
        targetStorageName = configurationHelper.getProperty(ConfigurationValue.S3_EXTERNAL_STORAGE_NAME_DEFAULT);
    }
    StorageEntity targetStorageEntity = storageDaoHelper.getStorageEntity(targetStorageName);
    assertTargetStorageEntityValid(targetStorageEntity);
    // Generate a random UUID value.
    String uuid = UUID.randomUUID().toString();
    // Create business object data key with partition value set to the generated UUID.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey().getNamespace(), uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey().getBusinessObjectDefinitionName(), uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey().getBusinessObjectFormatUsage(), uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey().getBusinessObjectFormatFileType(), uploadSingleInitiationRequest.getSourceBusinessObjectFormatKey().getBusinessObjectFormatVersion(), uuid, null, BusinessObjectDataEntity.BUSINESS_OBJECT_DATA_INITIAL_VERSION);
    // Get a file upload specific S3 key prefix for the source storage based on the generated UUID.
    String sourceStorageDirectoryPath = s3KeyPrefixHelper.buildS3KeyPrefix(sourceStorageEntity, sourceBusinessObjectFormatEntity, businessObjectDataKey);
    String sourceStorageFilePath = String.format("%s/%s", sourceStorageDirectoryPath, uploadSingleInitiationRequest.getFile().getFileName());
    // Create a business object data create request.
    BusinessObjectDataCreateRequest sourceBusinessObjectDataCreateRequest = businessObjectDataHelper.createBusinessObjectDataCreateRequest(sourceBusinessObjectFormatEntity, uuid, BusinessObjectDataStatusEntity.UPLOADING, uploadSingleInitiationRequest.getBusinessObjectDataAttributes(), sourceStorageEntity, sourceStorageDirectoryPath, sourceStorageFilePath, uploadSingleInitiationRequest.getFile().getFileSizeBytes(), null);
    // Create a new business object data instance. Set the flag to false, since for the file upload service the file size value is optional.
    BusinessObjectData sourceBusinessObjectData = businessObjectDataDaoHelper.createBusinessObjectData(sourceBusinessObjectDataCreateRequest, false);
    // Get a file upload specific S3 key prefix for the target storage based on the generated UUID.
    String targetStorageDirectoryPath = s3KeyPrefixHelper.buildS3KeyPrefix(targetStorageEntity, targetBusinessObjectFormatEntity, businessObjectDataKey);
    String targetStorageFilePath = String.format("%s/%s", targetStorageDirectoryPath, uploadSingleInitiationRequest.getFile().getFileName());
    uploadDownloadHelperService.assertS3ObjectKeyDoesNotExist(storageHelper.getStorageBucketName(targetStorageEntity), targetStorageFilePath);
    // Create a target business object data based on the source business object data and target business object format.
    BusinessObjectDataCreateRequest targetBusinessObjectDataCreateRequest = businessObjectDataHelper.createBusinessObjectDataCreateRequest(targetBusinessObjectFormatEntity, uuid, BusinessObjectDataStatusEntity.UPLOADING, uploadSingleInitiationRequest.getBusinessObjectDataAttributes(), targetStorageEntity, targetStorageDirectoryPath, targetStorageFilePath, uploadSingleInitiationRequest.getFile().getFileSizeBytes(), null);
    // Create a target business object data instance. Set the flag to false, since for the file upload service the file size value is optional.
    BusinessObjectData targetBusinessObjectData = businessObjectDataDaoHelper.createBusinessObjectData(targetBusinessObjectDataCreateRequest, false);
    // Get decrypted AWS ARN of the role that is required to provide access to S3_MANAGED_LOADING_DOCK storage.
    String awsRoleArn = getStorageUploadRoleArn(sourceStorageEntity);
    // Get expiration interval for the pre-signed URL to be generated.
    Integer awsRoleDurationSeconds = getStorageUploadSessionDuration(sourceStorageEntity);
    String awsKmsKeyId = storageHelper.getStorageKmsKeyId(sourceStorageEntity);
    // Get the temporary security credentials to access S3_MANAGED_STORAGE.
    Credentials assumedSessionCredentials = stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), String.valueOf(sourceBusinessObjectData.getId()), awsRoleArn, awsRoleDurationSeconds, createUploaderPolicy(s3BucketName, sourceStorageFilePath, awsKmsKeyId));
    // Create the response.
    UploadSingleInitiationResponse response = new UploadSingleInitiationResponse();
    response.setSourceBusinessObjectData(sourceBusinessObjectData);
    response.setTargetBusinessObjectData(targetBusinessObjectData);
    response.setFile(uploadSingleInitiationRequest.getFile());
    response.setUuid(uuid);
    response.setAwsAccessKey(assumedSessionCredentials.getAccessKeyId());
    response.setAwsSecretKey(assumedSessionCredentials.getSecretAccessKey());
    response.setAwsSessionToken(assumedSessionCredentials.getSessionToken());
    response.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(assumedSessionCredentials.getExpiration()));
    response.setAwsKmsKeyId(awsKmsKeyId);
    response.setTargetStorageName(targetStorageName);
    return response;
}

Example 14 with UploadSingleInitiationResponse

use of org.finra.herd.model.api.xml.UploadSingleInitiationResponse in project herd by FINRAOS.

the class UploadDownloadServiceTest method testExtendUploadSingleCredentials.

@Test
public void testExtendUploadSingleCredentials() throws InterruptedException {
    // Create source and target business object formats database entities which are required to initiate an upload.
    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();
    // Initiate a file upload.
    UploadSingleInitiationResponse uploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest());
    // Sleep a short amount of time to ensure the extended credentials don't return the same expiration as the initial credentials.
    Thread.sleep(10);
    // Initiate the download against the uploaded data (i.e. the target business object data).
    UploadSingleCredentialExtensionResponse uploadSingleCredentialExtensionResponse = extendUploadSingleCredentials(uploadSingleInitiationResponse.getSourceBusinessObjectData());
    // Validate the returned object.
    assertNotNull(uploadSingleCredentialExtensionResponse.getAwsAccessKey());
    assertNotNull(uploadSingleCredentialExtensionResponse.getAwsSecretKey());
    assertNotNull(uploadSingleCredentialExtensionResponse.getAwsSessionToken());
    assertNotNull(uploadSingleCredentialExtensionResponse.getAwsSessionExpirationTime());
    assertNotNull(uploadSingleInitiationResponse.getAwsSessionExpirationTime());
    // We are displaying the values in case there is a problem because this test was acting flaky.
    if (uploadSingleCredentialExtensionResponse.getAwsSessionExpirationTime().toGregorianCalendar().getTimeInMillis() <= uploadSingleInitiationResponse.getAwsSessionExpirationTime().toGregorianCalendar().getTimeInMillis()) {
        fail("Initial expiration time \"" + uploadSingleInitiationResponse.getAwsSessionExpirationTime().toGregorianCalendar().getTimeInMillis() + "\" is not > extended expiration time \"" + uploadSingleCredentialExtensionResponse.getAwsSessionExpirationTime().toGregorianCalendar().getTimeInMillis() + "\".");
    }
}

Example 15 with UploadSingleInitiationResponse

use of org.finra.herd.model.api.xml.UploadSingleInitiationResponse in project herd by FINRAOS.

the class UploadDownloadServiceTest method testInitiateUploadSingleAssertUseTargetStorageInRequest.

/**
 * Asserts that the target business object data that is created is using the target storage name that is specified in the request.
 */
@Test
public void testInitiateUploadSingleAssertUseTargetStorageInRequest() {
    // Create database entities required for testing.
    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME_3);
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), "testBucketName"));
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), "$environment/$namespace/$businessObjectDataPartitionValue"));
    storageEntity.getAttributes().add(storageDaoTestHelper.createStorageAttributeEntity(storageEntity, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"));
    // Initiate a file upload.
    UploadSingleInitiationRequest uploadSingleInitiationRequest = uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest();
    uploadSingleInitiationRequest.setTargetStorageName(STORAGE_NAME_3);
    UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadSingleInitiationRequest);
    // Validate the returned object.
    uploadDownloadServiceTestHelper.validateUploadSingleInitiationResponse(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, businessObjectDefinitionServiceTestHelper.getNewAttributes(), FILE_NAME, FILE_SIZE_1_KB, STORAGE_NAME_3, resultUploadSingleInitiationResponse);
    BusinessObjectDataEntity targetBusinessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, resultUploadSingleInitiationResponse.getTargetBusinessObjectData().getPartitionValue(), null, 0));
    assertNotNull(targetBusinessObjectDataEntity);
    assertNotNull(targetBusinessObjectDataEntity.getStorageUnits());
    assertEquals(1, targetBusinessObjectDataEntity.getStorageUnits().size());
    StorageUnitEntity storageUnit = IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0);
    assertNotNull(storageUnit);
    assertNotNull(storageUnit.getStorage());
    assertEquals(STORAGE_NAME_3, storageUnit.getStorage().getName());
}