Examples with S3FileTransferResultsDto org.finra.herd.model.dto.S3FileTransferResultsDto used on opensource projects

Example 26 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3ServiceTest method testUploadDirectory.

@Test
public void testUploadDirectory() throws InterruptedException {
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer result DTO.
    S3FileTransferResultsDto s3FileTransferResultsDto = new S3FileTransferResultsDto();
    // Mock the external calls.
    when(s3Dao.uploadDirectory(s3FileTransferRequestParamsDto)).thenReturn(s3FileTransferResultsDto);
    // Call the method under test.
    S3FileTransferResultsDto result = s3Service.uploadDirectory(s3FileTransferRequestParamsDto);
    // Verify the external calls.
    verify(s3Dao).uploadDirectory(s3FileTransferRequestParamsDto);
    verifyNoMoreInteractions(s3Dao);
    // Validate the returned object.
    assertEquals(s3FileTransferResultsDto, result);
}

Example 27 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3ServiceTest method testCopyFile.

@Test
public void testCopyFile() throws InterruptedException {
    // Create an S3 file copy request parameters DTO.
    S3FileCopyRequestParamsDto s3FileCopyRequestParamsDto = new S3FileCopyRequestParamsDto();
    // Create an S3 file transfer result DTO.
    S3FileTransferResultsDto s3FileTransferResultsDto = new S3FileTransferResultsDto();
    // Mock the external calls.
    when(s3Dao.copyFile(s3FileCopyRequestParamsDto)).thenReturn(s3FileTransferResultsDto);
    // Call the method under test.
    S3FileTransferResultsDto result = s3Service.copyFile(s3FileCopyRequestParamsDto);
    // Verify the external calls.
    verify(s3Dao).copyFile(s3FileCopyRequestParamsDto);
    verifyNoMoreInteractions(s3Dao);
    // Validate the returned object.
    assertEquals(s3FileTransferResultsDto, result);
}

Example 28 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class AbstractDataBridgeTest method uploadTestDataFilesToS3.

/**
 * Creates locally specified list of files and uploads them to the test S3 bucket. This method also creates 0 byte S3 directory markers relative to the s3
 * key prefix.
 *
 * @param s3KeyPrefix the destination S3 key prefix
 * @param manifestFiles the list of test data files to be created and uploaded to S3
 * @param directoryPaths the list of directory paths to be created in S3 relative to the S3 key prefix
 * <p/>
 * TODO: This method is basically a copy of prepareTestS3Files() from BusinessObjectDataServiceCreateBusinessObjectDataTest.java, so they both should be
 * replaced by a common helper method in AbstractDaoTest.java (a common parent class).
 */
protected void uploadTestDataFilesToS3(String s3KeyPrefix, List<ManifestFile> manifestFiles, List<String> directoryPaths) throws Exception {
    // Create local test data files.
    createTestDataFiles(LOCAL_TEMP_PATH_INPUT, manifestFiles);
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    String s3KeyPrefixWithTrailingSlash = s3KeyPrefix + "/";
    // Upload test file to S3.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixWithTrailingSlash);
    s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_INPUT.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Service.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate the transfer result.
    assertEquals(Long.valueOf(manifestFiles.size()), results.getTotalFilesTransferred());
    // Create 0 byte S3 directory markers.
    for (String directoryPath : directoryPaths) {
        // Create 0 byte directory marker.
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/" + directoryPath);
        s3Service.createDirectory(s3FileTransferRequestParamsDto);
    }
    // Restore the S3 key prefix value.
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixWithTrailingSlash);
    // Validate the uploaded S3 files and created directory markers, if any.
    assertEquals(manifestFiles.size() + directoryPaths.size(), s3Service.listDirectory(s3FileTransferRequestParamsDto).size());
}

Example 29 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class DownloaderControllerTest method testPerformDownloadAssertCleanTargetDirectoryWhenError.

/**
 * Asserts that the target directory is cleared (ie. all files under the directory is removed recursively) when there is an error during download.
 */
@Test
public void testPerformDownloadAssertCleanTargetDirectoryWhenError() throws Exception {
    /*
         * Create and inject mock objects
         */
    DownloaderWebClient mockDownloaderWebClient = mock(DownloaderWebClient.class);
    DownloaderWebClient originalDownloaderWebClient = (DownloaderWebClient) ReflectionTestUtils.getField(downloaderController, "downloaderWebClient");
    ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", mockDownloaderWebClient);
    DownloaderManifestReader mockDownloaderManifestReader = mock(DownloaderManifestReader.class);
    DownloaderManifestReader originalDownloaderManifestReader = (DownloaderManifestReader) ReflectionTestUtils.getField(downloaderController, "manifestReader");
    ReflectionTestUtils.setField(downloaderController, "manifestReader", mockDownloaderManifestReader);
    BusinessObjectDataHelper mockBusinessObjectDataHelper = mock(BusinessObjectDataHelper.class);
    BusinessObjectDataHelper originalBusinessObjectDataHelper = (BusinessObjectDataHelper) ReflectionTestUtils.getField(downloaderController, "businessObjectDataHelper");
    ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", mockBusinessObjectDataHelper);
    S3Service mockS3Service = mock(S3Service.class);
    S3Service originalS3Service = (S3Service) ReflectionTestUtils.getField(downloaderController, "s3Service");
    ReflectionTestUtils.setField(downloaderController, "s3Service", mockS3Service);
    StorageFileHelper mockStorageFileHelper = mock(StorageFileHelper.class);
    StorageFileHelper originalStorageFileHelper = (StorageFileHelper) ReflectionTestUtils.getField(downloaderController, "storageFileHelper");
    ReflectionTestUtils.setField(downloaderController, "storageFileHelper", mockStorageFileHelper);
    StorageHelper mockStorageHelper = mock(StorageHelper.class);
    StorageHelper originalStorageHelper = (StorageHelper) ReflectionTestUtils.getField(downloaderController, "storageHelper");
    ReflectionTestUtils.setField(downloaderController, "storageHelper", mockStorageHelper);
    /*
         * Start test
         */
    Path localPath = Files.createTempDirectory(null);
    try {
        String s3KeyPrefix = "s3KeyPrefix";
        String storageName = "storageName";
        IOException expectedException = new IOException();
        Path targetDirectoryPath = localPath.resolve(s3KeyPrefix);
        DownloaderInputManifestDto downloaderInputManifestDto = new DownloaderInputManifestDto();
        BusinessObjectData businessObjectData = new BusinessObjectData();
        StorageUnit storageUnit = new StorageUnit(new Storage(storageName, null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
        S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation();
        s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix);
        /*
             * Mock operations on mocked dependencies
             */
        when(mockDownloaderManifestReader.readJsonManifest(any())).thenReturn(downloaderInputManifestDto);
        when(mockDownloaderWebClient.getBusinessObjectData(any())).thenReturn(businessObjectData);
        when(mockBusinessObjectDataHelper.getStorageUnitByStorageName(any(), any())).thenReturn(storageUnit);
        when(mockDownloaderWebClient.getS3KeyPrefix(any())).thenReturn(s3KeyPrefixInformation);
        when(mockS3Service.downloadDirectory(any())).then(new Answer<S3FileTransferResultsDto>() {

            @Override
            public S3FileTransferResultsDto answer(InvocationOnMock invocation) throws Throwable {
                Files.createFile(targetDirectoryPath.resolve("file"));
                throw expectedException;
            }
        });
        /*
             * Make the call to the method under test
             */
        RegServerAccessParamsDto regServerAccessParamsDto = null;
        File manifestPath = null;
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setLocalPath(localPath.toString());
        s3FileTransferRequestParamsDto.setMaxThreads(1);
        try {
            downloaderController.performDownload(regServerAccessParamsDto, manifestPath, s3FileTransferRequestParamsDto);
            // Expect an exception, fail if no exception
            fail();
        } catch (Exception e) {
            // Assert that the exception thrown by the mock is what is actually thrown
            assertEquals(expectedException, e);
            // Assert that the target directory is cleaned
            assertEquals(0, targetDirectoryPath.toFile().list().length);
        }
    } finally {
        /*
             * Restore mocked dependencies to their original implementation
             */
        ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", originalDownloaderWebClient);
        ReflectionTestUtils.setField(downloaderController, "manifestReader", originalDownloaderManifestReader);
        ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", originalBusinessObjectDataHelper);
        ReflectionTestUtils.setField(downloaderController, "s3Service", originalS3Service);
        ReflectionTestUtils.setField(downloaderController, "storageFileHelper", originalStorageFileHelper);
        ReflectionTestUtils.setField(downloaderController, "storageHelper", originalStorageHelper);
        // Clean up any temporary files
        FileUtils.deleteDirectory(localPath.toFile());
    }
}

Example 30 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class DownloaderControllerTest method testLogLocalDirectoryContents.

@Test
public void testLogLocalDirectoryContents() throws Exception {
    String appenderName = "TestWriterAppender";
    StringWriter stringWriter = addLoggingWriterAppender(appenderName);
    LogLevel originalLevel = getLogLevel(DownloaderController.class);
    setLogLevel(DownloaderController.class, LogLevel.INFO);
    /*
         * Create and inject mock objects
         */
    DownloaderWebClient mockDownloaderWebClient = mock(DownloaderWebClient.class);
    DownloaderWebClient originalDownloaderWebClient = (DownloaderWebClient) ReflectionTestUtils.getField(downloaderController, "downloaderWebClient");
    ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", mockDownloaderWebClient);
    DownloaderManifestReader mockDownloaderManifestReader = mock(DownloaderManifestReader.class);
    DownloaderManifestReader originalDownloaderManifestReader = (DownloaderManifestReader) ReflectionTestUtils.getField(downloaderController, "manifestReader");
    ReflectionTestUtils.setField(downloaderController, "manifestReader", mockDownloaderManifestReader);
    BusinessObjectDataHelper mockBusinessObjectDataHelper = mock(BusinessObjectDataHelper.class);
    BusinessObjectDataHelper originalBusinessObjectDataHelper = (BusinessObjectDataHelper) ReflectionTestUtils.getField(downloaderController, "businessObjectDataHelper");
    ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", mockBusinessObjectDataHelper);
    S3Service mockS3Service = mock(S3Service.class);
    S3Service originalS3Service = (S3Service) ReflectionTestUtils.getField(downloaderController, "s3Service");
    ReflectionTestUtils.setField(downloaderController, "s3Service", mockS3Service);
    StorageFileHelper mockStorageFileHelper = mock(StorageFileHelper.class);
    StorageFileHelper originalStorageFileHelper = (StorageFileHelper) ReflectionTestUtils.getField(downloaderController, "storageFileHelper");
    ReflectionTestUtils.setField(downloaderController, "storageFileHelper", mockStorageFileHelper);
    StorageHelper mockStorageHelper = mock(StorageHelper.class);
    StorageHelper originalStorageHelper = (StorageHelper) ReflectionTestUtils.getField(downloaderController, "storageHelper");
    ReflectionTestUtils.setField(downloaderController, "storageHelper", mockStorageHelper);
    /*
         * Start test
         */
    Path localPath = Files.createTempDirectory(null);
    try {
        String s3KeyPrefix = "s3KeyPrefix";
        String storageName = "storageName";
        Path targetDirectoryPath = localPath.resolve(s3KeyPrefix);
        Path targetFilePath = targetDirectoryPath.resolve("file");
        DownloaderInputManifestDto downloaderInputManifestDto = new DownloaderInputManifestDto();
        BusinessObjectData businessObjectData = new BusinessObjectData();
        StorageUnit storageUnit = new StorageUnit(new Storage(storageName, null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null);
        S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation();
        s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix);
        /*
             * Mock operations on mocked dependencies
             */
        when(mockDownloaderManifestReader.readJsonManifest(any())).thenReturn(downloaderInputManifestDto);
        when(mockDownloaderWebClient.getBusinessObjectData(any())).thenReturn(businessObjectData);
        when(mockBusinessObjectDataHelper.getStorageUnitByStorageName(any(), any())).thenReturn(storageUnit);
        when(mockDownloaderWebClient.getS3KeyPrefix(any())).thenReturn(s3KeyPrefixInformation);
        when(mockS3Service.downloadDirectory(any())).then(new Answer<S3FileTransferResultsDto>() {

            @Override
            public S3FileTransferResultsDto answer(InvocationOnMock invocation) throws Throwable {
                Files.createFile(targetFilePath);
                return null;
            }
        });
        /*
             * Make the call to the method under test
             */
        RegServerAccessParamsDto regServerAccessParamsDto = null;
        File manifestPath = null;
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setLocalPath(localPath.toString());
        s3FileTransferRequestParamsDto.setMaxThreads(1);
        downloaderController.performDownload(regServerAccessParamsDto, manifestPath, s3FileTransferRequestParamsDto);
        assertEquals(String.format("Found 1 files in \"%s\" target local directory:%n    %s%n", targetDirectoryPath, targetFilePath), stringWriter.toString());
    } finally {
        setLogLevel(DownloaderController.class, originalLevel);
        removeLoggingAppender(appenderName);
        /*
             * Restore mocked dependencies to their original implementation
             */
        ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", originalDownloaderWebClient);
        ReflectionTestUtils.setField(downloaderController, "manifestReader", originalDownloaderManifestReader);
        ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", originalBusinessObjectDataHelper);
        ReflectionTestUtils.setField(downloaderController, "s3Service", originalS3Service);
        ReflectionTestUtils.setField(downloaderController, "storageFileHelper", originalStorageFileHelper);
        ReflectionTestUtils.setField(downloaderController, "storageHelper", originalStorageHelper);
        // Clean up any temporary files
        FileUtils.deleteDirectory(localPath.toFile());
    }
}