Examples with AwsCredential org.finra.herd.model.api.xml.AwsCredential used on opensource projects

Example 11 with AwsCredential

use of org.finra.herd.model.api.xml.AwsCredential in project herd by FINRAOS.

the class StorageUnitServiceImpl method getBusinessObjectDataS3Credential.

/**
 * Creates and returns a set of AWS credentials which can be used to access the S3 object indicated by the given business object data and storage.
 *
 * @param businessObjectDataKey Business object data key
 * @param createNewVersion true to create credentials for the next version up from the latest business object data, otherwise, uses specified data version
 * in data key.
 * @param storageName Name of storage to access
 * @param isUpload true if this credential is to upload, false to download
 *
 * @return Credentials which has the permissions to perform the specified actions at the specified storage.
 */
private AwsCredential getBusinessObjectDataS3Credential(BusinessObjectDataKey businessObjectDataKey, Boolean createNewVersion, String storageName, boolean isUpload) {
    Assert.isTrue(StringUtils.isNotBlank(storageName), "storageName must be specified");
    Assert.isTrue(businessObjectDataKey.getBusinessObjectDataVersion() != null || createNewVersion != null, "One of businessObjectDataVersion or createNewVersion must be specified.");
    Assert.isTrue(businessObjectDataKey.getBusinessObjectDataVersion() == null || !Boolean.TRUE.equals(createNewVersion), "createNewVersion must be false or unspecified when businessObjectDataVersion is specified.");
    /*
         * Choose configurations based on whether this is an upload or download operation.
         */
    ConfigurationValue roleArnConfigurationValue;
    ConfigurationValue defaultSessionDurationConfigurationValue;
    ConfigurationValue sessionDurationConfigurationValue;
    S3Actions[] s3Actions;
    KmsActions[] kmsActions;
    if (isUpload) {
        roleArnConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_UPLOAD_ROLE_ARN;
        defaultSessionDurationConfigurationValue = ConfigurationValue.AWS_S3_DEFAULT_UPLOAD_SESSION_DURATION_SECS;
        sessionDurationConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_UPLOAD_SESSION_DURATION_SECS;
        s3Actions = new S3Actions[] { S3Actions.PutObject, S3Actions.DeleteObject };
        kmsActions = new KmsActions[] { KmsActions.GENERATE_DATA_KEY, KmsActions.DECRYPT };
    } else {
        roleArnConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_DOWNLOAD_ROLE_ARN;
        defaultSessionDurationConfigurationValue = ConfigurationValue.AWS_S3_DEFAULT_DOWNLOAD_SESSION_DURATION_SECS;
        sessionDurationConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_DOWNLOAD_SESSION_DURATION_SECS;
        s3Actions = new S3Actions[] { S3Actions.GetObject };
        kmsActions = new KmsActions[] { KmsActions.DECRYPT };
    }
    StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageName.trim());
    String roleArn = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(roleArnConfigurationValue), storageEntity, true);
    Integer durationSeconds = storageHelper.getStorageAttributeIntegerValueByName(configurationHelper.getProperty(sessionDurationConfigurationValue), storageEntity, configurationHelper.getProperty(defaultSessionDurationConfigurationValue, Integer.class));
    String bucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageEntity, true);
    S3KeyPrefixInformation s3KeyPrefixInformation = getS3KeyPrefixImpl(businessObjectDataKey, null, storageName, createNewVersion);
    /*
         * Policy is different based on whether this is meant for downloading or uploading.
         * However, both uploader and downloader requires a ListBucket at the bucket level.
         */
    AwsPolicyBuilder awsPolicyBuilder = new AwsPolicyBuilder().withS3Prefix(bucketName, s3KeyPrefixInformation.getS3KeyPrefix(), s3Actions).withS3(bucketName, null, S3Actions.ListObjects);
    /*
         * Only add KMS policies if the storage specifies a KMS ID
         */
    String kmsKeyId = getStorageKmsKeyId(storageEntity);
    if (kmsKeyId != null) {
        awsPolicyBuilder.withKms(kmsKeyId.trim(), kmsActions);
    }
    Credentials credentials = stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), UUID.randomUUID().toString(), roleArn, durationSeconds, awsPolicyBuilder.build());
    AwsCredential awsCredential = new AwsCredential();
    awsCredential.setAwsAccessKey(credentials.getAccessKeyId());
    awsCredential.setAwsSecretKey(credentials.getSecretAccessKey());
    awsCredential.setAwsSessionToken(credentials.getSessionToken());
    awsCredential.setAwsSessionExpirationTime(HerdDateUtils.getXMLGregorianCalendarValue(credentials.getExpiration()));
    return awsCredential;
}

Example 12 with AwsCredential

use of org.finra.herd.model.api.xml.AwsCredential in project herd by FINRAOS.

the class DownloaderControllerTest method testPerformDownloadAssertCredentialsRetrieved.

/**
 * Asserts that the controller is sending the proper implementation of credentials provider when calling S3.
 */
@Test
public void testPerformDownloadAssertCredentialsRetrieved() throws Exception {
    /*
         * Create and inject mock objects
         */
    DownloaderWebClient mockDownloaderWebClient = mock(DownloaderWebClient.class);
    DownloaderWebClient originalDownloaderWebClient = (DownloaderWebClient) ReflectionTestUtils.getField(downloaderController, "downloaderWebClient");
    ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", mockDownloaderWebClient);
    DownloaderManifestReader mockDownloaderManifestReader = mock(DownloaderManifestReader.class);
    DownloaderManifestReader originalDownloaderManifestReader = (DownloaderManifestReader) ReflectionTestUtils.getField(downloaderController, "manifestReader");
    ReflectionTestUtils.setField(downloaderController, "manifestReader", mockDownloaderManifestReader);
    BusinessObjectDataHelper mockBusinessObjectDataHelper = mock(BusinessObjectDataHelper.class);
    BusinessObjectDataHelper originalBusinessObjectDataHelper = (BusinessObjectDataHelper) ReflectionTestUtils.getField(downloaderController, "businessObjectDataHelper");
    ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", mockBusinessObjectDataHelper);
    S3Service mockS3Service = mock(S3Service.class);
    S3Service originalS3Service = (S3Service) ReflectionTestUtils.getField(downloaderController, "s3Service");
    ReflectionTestUtils.setField(downloaderController, "s3Service", mockS3Service);
    StorageFileHelper mockStorageFileHelper = mock(StorageFileHelper.class);
    StorageFileHelper originalStorageFileHelper = (StorageFileHelper) ReflectionTestUtils.getField(downloaderController, "storageFileHelper");
    ReflectionTestUtils.setField(downloaderController, "storageFileHelper", mockStorageFileHelper);
    StorageHelper mockStorageHelper = mock(StorageHelper.class);
    StorageHelper originalStorageHelper = (StorageHelper) ReflectionTestUtils.getField(downloaderController, "storageHelper");
    ReflectionTestUtils.setField(downloaderController, "storageHelper", mockStorageHelper);
    /*
         * Start test
         */
    Path localPath = Files.createTempDirectory(null);
    try {
        String s3KeyPrefix = "s3KeyPrefix";
        String storageName = "storageName";
        DownloaderInputManifestDto downloaderInputManifestDto = new DownloaderInputManifestDto();
        downloaderInputManifestDto.setStorageName(storageName);
        BusinessObjectData businessObjectData = new BusinessObjectData();
        StorageUnit storageUnit = new StorageUnit(new Storage(storageName, null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
        S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation();
        s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix);
        /*
             * Mock operations on mocked dependencies
             */
        when(mockDownloaderManifestReader.readJsonManifest(any())).thenReturn(downloaderInputManifestDto);
        when(mockDownloaderWebClient.getBusinessObjectData(any())).thenReturn(businessObjectData);
        when(mockBusinessObjectDataHelper.getStorageUnitByStorageName(any(), any())).thenReturn(storageUnit);
        when(mockDownloaderWebClient.getS3KeyPrefix(any())).thenReturn(s3KeyPrefixInformation);
        when(mockDownloaderWebClient.getStorageUnitDownloadCredential(any(), any())).thenReturn(new StorageUnitDownloadCredential(new AwsCredential("awsAccessKey", "awsSecretKey", "awsSessionToken", DatatypeFactory.newInstance().newXMLGregorianCalendar())));
        when(mockS3Service.downloadDirectory(any())).then(new Answer<S3FileTransferResultsDto>() {

            @Override
            public S3FileTransferResultsDto answer(InvocationOnMock invocation) throws Throwable {
                /*
                     * Call the providers' getAwsCredentials(), just like real implementation would.
                     */
                S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = invocation.getArgument(0);
                List<HerdAWSCredentialsProvider> additionalAwsCredentialsProviders = s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders();
                for (HerdAWSCredentialsProvider herdAWSCredentialsProvider : additionalAwsCredentialsProviders) {
                    herdAWSCredentialsProvider.getAwsCredential();
                }
                return null;
            }
        });
        /*
             * Make the call to the method under test
             */
        RegServerAccessParamsDto regServerAccessParamsDto = null;
        File manifestPath = null;
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setLocalPath(localPath.toString());
        s3FileTransferRequestParamsDto.setMaxThreads(1);
        downloaderController.performDownload(regServerAccessParamsDto, manifestPath, s3FileTransferRequestParamsDto);
        // Assert that the proper delegate method is called with the expected params to retrieve credentials
        verify(mockDownloaderWebClient).getStorageUnitDownloadCredential(downloaderInputManifestDto, storageName);
    } finally {
        /*
             * Restore mocked dependencies to their original implementation
             */
        ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", originalDownloaderWebClient);
        ReflectionTestUtils.setField(downloaderController, "manifestReader", originalDownloaderManifestReader);
        ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", originalBusinessObjectDataHelper);
        ReflectionTestUtils.setField(downloaderController, "s3Service", originalS3Service);
        ReflectionTestUtils.setField(downloaderController, "storageFileHelper", originalStorageFileHelper);
        ReflectionTestUtils.setField(downloaderController, "storageHelper", originalStorageHelper);
        // Clean up any temporary files
        FileUtils.deleteDirectory(localPath.toFile());
    }
}

Example 13 with AwsCredential

use of org.finra.herd.model.api.xml.AwsCredential in project herd by FINRAOS.

the class DownloaderWebClientTest method testGetBusinessObjectDataDownloadCredential1.

@Test
public void testGetBusinessObjectDataDownloadCredential1() throws Exception {
    DownloaderInputManifestDto manifest = new DownloaderInputManifestDto();
    manifest.setNamespace("test1");
    manifest.setBusinessObjectDefinitionName("test2");
    manifest.setBusinessObjectFormatUsage("test3");
    manifest.setBusinessObjectFormatFileType("test4");
    manifest.setBusinessObjectFormatVersion("test5");
    manifest.setPartitionValue("test6");
    manifest.setSubPartitionValues(Arrays.asList("test7", "test8"));
    manifest.setBusinessObjectDataVersion("test9");
    String storageName = "test10";
    StorageUnitDownloadCredential storageUnitDownloadCredential = downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName);
    Assert.assertNotNull(storageUnitDownloadCredential);
    AwsCredential awsCredential = storageUnitDownloadCredential.getAwsCredential();
    Assert.assertNotNull(awsCredential);
    Assert.assertEquals("https://testWebServiceHostname:1234/herd-app/rest/storageUnits/download/credential/namespaces/test1" + "/businessObjectDefinitionNames/test2/businessObjectFormatUsages/test3/businessObjectFormatFileTypes/test4/businessObjectFormatVersions/test5" + "/partitionValues/test6/businessObjectDataVersions/test9/storageNames/test10?subPartitionValues=test7%7Ctest8", awsCredential.getAwsAccessKey());
}

Example 14 with AwsCredential

use of org.finra.herd.model.api.xml.AwsCredential in project herd by FINRAOS.

the class DownloaderWebClientTest method testGetBusinessObjectDataDownloadCredential2.

@Test
public void testGetBusinessObjectDataDownloadCredential2() throws Exception {
    DownloaderInputManifestDto manifest = new DownloaderInputManifestDto();
    manifest.setNamespace("test1");
    manifest.setBusinessObjectDefinitionName("test2");
    manifest.setBusinessObjectFormatUsage("test3");
    manifest.setBusinessObjectFormatFileType("test4");
    manifest.setBusinessObjectFormatVersion("test5");
    manifest.setPartitionValue("test6");
    manifest.setBusinessObjectDataVersion("test9");
    String storageName = "test10";
    downloaderWebClient.getRegServerAccessParamsDto().setUseSsl(true);
    StorageUnitDownloadCredential storageUnitDownloadCredential = downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName);
    Assert.assertNotNull(storageUnitDownloadCredential);
    AwsCredential awsCredential = storageUnitDownloadCredential.getAwsCredential();
    Assert.assertNotNull(awsCredential);
    Assert.assertEquals("https://testWebServiceHostname:1234/herd-app/rest/storageUnits/download/credential/namespaces/test1" + "/businessObjectDefinitionNames/test2/businessObjectFormatUsages/test3/businessObjectFormatFileTypes/test4/businessObjectFormatVersions/test5" + "/partitionValues/test6/businessObjectDataVersions/test9/storageNames/test10", awsCredential.getAwsAccessKey());
}

Example 15 with AwsCredential

use of org.finra.herd.model.api.xml.AwsCredential in project herd by FINRAOS.

the class DownloaderController method performDownload.

/**
 * Executes the downloader workflow.
 *
 * @param regServerAccessParamsDto the DTO for the parameters required to communicate with the herd registration server
 * @param manifestPath the local path to the manifest file
 * @param s3FileTransferRequestParamsDto the S3 file transfer DTO request parameters
 *
 * @throws InterruptedException if the upload thread was interrupted.
 * @throws JAXBException if a JAXB error was encountered.
 * @throws IOException if an I/O error was encountered.
 * @throws URISyntaxException if a URI syntax error was encountered.
 */
@SuppressFBWarnings(value = { "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" }, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.")
public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto) throws InterruptedException, JAXBException, IOException, URISyntaxException {
    boolean cleanUpTargetLocalDirectoryOnFailure = false;
    File targetLocalDirectory = null;
    try {
        // Process manifest file.
        DownloaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
        String storageName = getStorageNameFromManifest(manifest);
        // Get business object data from the herd registration server.
        downloaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
        BusinessObjectData businessObjectData = downloaderWebClient.getBusinessObjectData(manifest);
        manifest.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
        manifest.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
        s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() {

            @Override
            public AwsCredential getNewAwsCredential() throws Exception {
                return downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName).getAwsCredential();
            }
        });
        // Get a storage unit that belongs to the S3 storage.
        StorageUnit storageUnit = businessObjectDataHelper.getStorageUnitByStorageName(businessObjectData, storageName);
        // Get the expected S3 key prefix and S3 bucket name.
        S3KeyPrefixInformation s3KeyPrefixInformation = downloaderWebClient.getS3KeyPrefix(businessObjectData);
        // Check if the target folder (local directory + S3 key prefix) exists and try to create it if it does not.
        targetLocalDirectory = Paths.get(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix()).toFile();
        if (!targetLocalDirectory.isDirectory()) {
            // Create the local directory including any necessary but nonexistent parent directories.
            if (!targetLocalDirectory.mkdirs()) {
                throw new IllegalArgumentException(String.format("Failed to create target local directory \"%s\".", targetLocalDirectory.getPath()));
            }
        } else {
            // Check if the target local directory is empty.
            if (targetLocalDirectory.list().length > 0) {
                throw new IllegalArgumentException(String.format("The target local directory \"%s\" is not empty.", targetLocalDirectory.getPath()));
            }
        }
        // Get S3 bucket information.
        Storage storage = downloaderWebClient.getStorage(storageName);
        // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
        String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
        // Get the list of S3 files matching the expected S3 key prefix.
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixInformation.getS3KeyPrefix() + "/");
        // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
        List<String> actualS3Files = storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Service.listDirectory(s3FileTransferRequestParamsDto, true));
        // Validate S3 files before we start the download.
        storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, s3KeyPrefixInformation.getS3KeyPrefix());
        // Special handling for the maxThreads command line option.
        s3FileTransferRequestParamsDto.setMaxThreads(adjustIntegerValue(s3FileTransferRequestParamsDto.getMaxThreads(), MIN_THREADS, MAX_THREADS));
        // Download S3 files to the target local directory.
        s3FileTransferRequestParamsDto.setRecursive(true);
        cleanUpTargetLocalDirectoryOnFailure = true;
        s3Service.downloadDirectory(s3FileTransferRequestParamsDto);
        // Validate the downloaded files.
        storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit);
        // Log a list of files downloaded to the target local directory.
        if (LOGGER.isInfoEnabled()) {
            logLocalDirectoryContents(targetLocalDirectory);
        }
        // Create a downloader output manifest file.
        DownloaderOutputManifestDto downloaderOutputManifestDto = createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefixInformation.getS3KeyPrefix());
        manifestWriter.writeJsonManifest(targetLocalDirectory, OUTPUT_MANIFEST_FILE_NAME, downloaderOutputManifestDto);
    } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) {
        // occurred, let's rollback the data transfer by cleaning up the local target directory.
        if (cleanUpTargetLocalDirectoryOnFailure) {
            LOGGER.info(String.format("Rolling back the S3 data transfer by cleaning up \"%s\" target local directory.", targetLocalDirectory));
            HerdFileUtils.cleanDirectoryIgnoreException(targetLocalDirectory);
        }
        throw e;
    }
}