Examples with S3Helper org.sagebionetworks.bridge.s3.S3Helper used on opensource projects

Example 6 with S3Helper

use of org.sagebionetworks.bridge.s3.S3Helper in project BridgeServer2 by Sage-Bionetworks.

the class UploadHandlersEndToEndTest method test.

private void test(UploadSchema schema, Survey survey, Map<String, String> fileMap, String nonZippedFileContent) throws Exception {
    // Set up zip service.
    UploadArchiveService unzipService = new UploadArchiveService();
    if (upload.isZipped()) {
        // fileMap is in strings. Convert to bytes so we can use the Zipper.
        Map<String, byte[]> fileBytesMap = new HashMap<>();
        for (Map.Entry<String, String> oneFile : fileMap.entrySet()) {
            String filename = oneFile.getKey();
            String fileString = oneFile.getValue();
            fileBytesMap.put(filename, fileString.getBytes(Charsets.UTF_8));
        }
        // Add metadata.json to the fileBytesMap.
        fileBytesMap.put("metadata.json", METADATA_JSON_CONTENT);
        // For zipping, we use the real service.
        unzipService.setMaxNumZipEntries(1000000);
        unzipService.setMaxZipEntrySize(1000000);
        rawFile = unzipService.zip(fileBytesMap);
    } else {
        // If it's not zipped, use the nonZippedFileContent as the raw file.
        rawFile = nonZippedFileContent.getBytes(Charsets.UTF_8);
    }
    // Set up UploadFileHelper
    DigestUtils mockMd5DigestUtils = mock(DigestUtils.class);
    when(mockMd5DigestUtils.digest(any(File.class))).thenReturn(TestConstants.MOCK_MD5);
    when(mockMd5DigestUtils.digest(any(byte[].class))).thenReturn(TestConstants.MOCK_MD5);
    UploadFileHelper uploadFileHelper = new UploadFileHelper();
    uploadFileHelper.setFileHelper(inMemoryFileHelper);
    uploadFileHelper.setMd5DigestUtils(mockMd5DigestUtils);
    uploadFileHelper.setS3Helper(mockS3UploadHelper);
    // set up S3DownloadHandler - mock S3 Helper
    // "S3" returns file unencrypted for simplicity of testing
    S3Helper mockS3DownloadHelper = mock(S3Helper.class);
    doAnswer(invocation -> {
        File destFile = invocation.getArgument(2);
        inMemoryFileHelper.writeBytes(destFile, rawFile);
        // Required return
        return null;
    }).when(mockS3DownloadHelper).downloadS3File(eq(TestConstants.UPLOAD_BUCKET), eq(UPLOAD_ID), any());
    S3DownloadHandler s3DownloadHandler = new S3DownloadHandler();
    s3DownloadHandler.setFileHelper(inMemoryFileHelper);
    s3DownloadHandler.setS3Helper(mockS3DownloadHelper);
    // set up DecryptHandler - For ease of tests, this will just return the input verbatim.
    UploadArchiveService mockUploadArchiveService = mock(UploadArchiveService.class);
    when(mockUploadArchiveService.decrypt(eq(TEST_APP_ID), any(InputStream.class))).thenAnswer(invocation -> invocation.getArgument(1));
    DecryptHandler decryptHandler = new DecryptHandler();
    decryptHandler.setFileHelper(inMemoryFileHelper);
    decryptHandler.setUploadArchiveService(mockUploadArchiveService);
    // Set up UnzipHandler
    UnzipHandler unzipHandler = new UnzipHandler();
    unzipHandler.setFileHelper(inMemoryFileHelper);
    unzipHandler.setUploadArchiveService(unzipService);
    // Set up InitRecordHandler
    InitRecordHandler initRecordHandler = new InitRecordHandler();
    initRecordHandler.setFileHelper(inMemoryFileHelper);
    // mock schema service
    UploadSchemaService mockUploadSchemaService = mock(UploadSchemaService.class);
    if (schema != null) {
        when(mockUploadSchemaService.getUploadSchemaByIdAndRev(TEST_APP_ID, schema.getSchemaId(), schema.getRevision())).thenReturn(schema);
        when(mockUploadSchemaService.getUploadSchemaByIdAndRevNoThrow(TEST_APP_ID, schema.getSchemaId(), schema.getRevision())).thenReturn(schema);
    }
    // mock survey service
    SurveyService mockSurveyService = mock(SurveyService.class);
    if (survey != null) {
        when(mockSurveyService.getSurvey(TEST_APP_ID, new GuidCreatedOnVersionHolderImpl(survey.getGuid(), survey.getCreatedOn()), false, true)).thenReturn(survey);
    }
    // set up IosSchemaValidationHandler
    IosSchemaValidationHandler2 iosSchemaValidationHandler = new IosSchemaValidationHandler2();
    iosSchemaValidationHandler.setFileHelper(inMemoryFileHelper);
    iosSchemaValidationHandler.setUploadFileHelper(uploadFileHelper);
    iosSchemaValidationHandler.setUploadSchemaService(mockUploadSchemaService);
    iosSchemaValidationHandler.setSurveyService(mockSurveyService);
    // set up GenericUploadFormatHandler
    GenericUploadFormatHandler genericUploadFormatHandler = new GenericUploadFormatHandler();
    genericUploadFormatHandler.setFileHelper(inMemoryFileHelper);
    genericUploadFormatHandler.setUploadFileHelper(uploadFileHelper);
    genericUploadFormatHandler.setUploadSchemaService(mockUploadSchemaService);
    genericUploadFormatHandler.setSurveyService(mockSurveyService);
    // set up UploadFormatHandler
    UploadFormatHandler uploadFormatHandler = new UploadFormatHandler();
    uploadFormatHandler.setV1LegacyHandler(iosSchemaValidationHandler);
    uploadFormatHandler.setV2GenericHandler(genericUploadFormatHandler);
    // set up StrictValidationHandler
    StrictValidationHandler strictValidationHandler = new StrictValidationHandler();
    strictValidationHandler.setUploadSchemaService(mockUploadSchemaService);
    AppService mockAppService = mock(AppService.class);
    when(mockAppService.getApp(TEST_APP_ID)).thenReturn(APP);
    strictValidationHandler.setAppService(mockAppService);
    Enrollment enrollment = Enrollment.create(TEST_APP_ID, "test-study", "userId", EXTERNAL_ID);
    // set up TranscribeConsentHandler
    Account account = Account.create();
    account.setDataGroups(ImmutableSet.of("parkinson", "test_user"));
    account.setSharingScope(SharingScope.SPONSORS_AND_PARTNERS);
    account.setEnrollments(ImmutableSet.of(enrollment));
    AccountService mockAccountService = mock(AccountService.class);
    when(mockAccountService.getAccount(any())).thenReturn(Optional.of(account));
    ParticipantService mockParticipantService = mock(ParticipantService.class);
    when(mockParticipantService.getStudyStartTime(any())).thenReturn(STUDY_START_TIME);
    TranscribeConsentHandler transcribeConsentHandler = new TranscribeConsentHandler();
    transcribeConsentHandler.setAccountService(mockAccountService);
    transcribeConsentHandler.setParticipantService(mockParticipantService);
    // Set up UploadRawZipHandler.
    UploadRawZipHandler uploadRawZipHandler = new UploadRawZipHandler();
    uploadRawZipHandler.setUploadFileHelper(uploadFileHelper);
    // set up UploadArtifactsHandler
    UploadArtifactsHandler uploadArtifactsHandler = new UploadArtifactsHandler();
    uploadArtifactsHandler.setHealthDataService(mockHealthDataService);
    // set up task factory
    List<UploadValidationHandler> handlerList = ImmutableList.of(s3DownloadHandler, decryptHandler, unzipHandler, initRecordHandler, uploadFormatHandler, strictValidationHandler, transcribeConsentHandler, uploadRawZipHandler, uploadArtifactsHandler);
    UploadValidationTaskFactory taskFactory = new UploadValidationTaskFactory();
    taskFactory.setFileHelper(inMemoryFileHelper);
    taskFactory.setHandlerList(handlerList);
    taskFactory.setUploadDao(mockUploadDao);
    taskFactory.setHealthDataService(mockHealthDataService);
    // create task, execute
    UploadValidationTask task = taskFactory.newTask(TEST_APP_ID, upload);
    task.run();
}

Example 7 with S3Helper

use of org.sagebionetworks.bridge.s3.S3Helper in project BridgeServer2 by Sage-Bionetworks.

the class S3DownloadHandlerTest method test.

@Test
public void test() throws Exception {
    // The handler is a simple pass-through to the S3Helper, so just test that execution flows through
    // to the service as expected.
    // inputs
    DynamoUpload2 upload2 = new DynamoUpload2();
    upload2.setUploadId("test-upload-id");
    UploadValidationContext ctx = new UploadValidationContext();
    ctx.setUpload(upload2);
    // Set up file helper and temp dir
    InMemoryFileHelper inMemoryFileHelper = new InMemoryFileHelper();
    File tmpDir = inMemoryFileHelper.createTempDir();
    ctx.setTempDir(tmpDir);
    // mock S3 helper
    S3Helper mockS3Helper = mock(S3Helper.class);
    doAnswer(invocation -> {
        File destFile = invocation.getArgument(2);
        inMemoryFileHelper.writeBytes(destFile, "test data".getBytes(Charsets.UTF_8));
        return null;
    }).when(mockS3Helper).downloadS3File(eq(TestConstants.UPLOAD_BUCKET), eq("test-upload-id"), any());
    // set up test handler
    S3DownloadHandler handler = new S3DownloadHandler();
    handler.setFileHelper(inMemoryFileHelper);
    handler.setS3Helper(mockS3Helper);
    // execute and validate
    handler.handle(ctx);
    byte[] dataFileContent = inMemoryFileHelper.getBytes(ctx.getDataFile());
    assertEquals(new String(dataFileContent, Charsets.UTF_8), "test data");
}

Example 8 with S3Helper

use of org.sagebionetworks.bridge.s3.S3Helper in project synapse by otto-de.

the class SnapshotReadServiceIntegrationTest method setUp.

@Before
public void setUp() {
    s3Helper = new S3Helper(s3Client);
    s3Helper.createBucket(S3_UTILS_TEST_BUCKET);
    s3Helper.deleteAllObjectsInBucket(S3_UTILS_TEST_BUCKET);
}

Example 9 with S3Helper

use of org.sagebionetworks.bridge.s3.S3Helper in project synapse by otto-de.

the class S3SnapshotMessageStoreAcceptanceTest method setup.

@Before
public void setup() throws IOException, URISyntaxException {
    deleteSnapshotFilesFromTemp();
    s3Helper = new S3Helper(s3Client);
    s3Helper.createBucket(INTEGRATION_TEST_BUCKET);
    s3Helper.deleteAllObjectsInBucket(INTEGRATION_TEST_BUCKET);
}