Examples with InputStreamWrapper org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper used on opensource projects

Example 1 with InputStreamWrapper

use of org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper in project data-transfer-project by google.

the class FlickrPhotosImporter method uploadPhoto.

private String uploadPhoto(PhotoModel photo, UUID jobId) throws IOException, FlickrException {
    InputStream inStream;
    if (photo.isInTempStore()) {
        final InputStreamWrapper streamWrapper = jobStore.getStream(jobId, photo.getFetchableUrl());
        inStream = streamWrapper.getStream();
    } else {
        inStream = imageStreamProvider.get(photo.getFetchableUrl());
    }
    String photoTitle = Strings.isNullOrEmpty(photo.getTitle()) ? "" : photo.getTitle();
    String photoDescription = cleanString(photo.getDescription());
    UploadMetaData uploadMetaData = new UploadMetaData().setAsync(false).setPublicFlag(false).setFriendFlag(false).setFamilyFlag(false).setTitle(photoTitle).setDescription(photoDescription);
    perUserRateLimiter.acquire();
    String uploadResult = uploader.upload(inStream, uploadMetaData);
    inStream.close();
    monitor.debug(() -> String.format("%s: Flickr importer uploading photo: %s", jobId, photo));
    return uploadResult;
}

Example 2 with InputStreamWrapper

use of org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper in project data-transfer-project by google.

the class KoofrPhotosImporterTest method testImportItemFromJobStore.

@Test
public void testImportItemFromJobStore() throws Exception {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] { 0, 1, 2, 3, 4 });
    when(client.ensureRootFolder()).thenReturn("/root");
    when(jobStore.getStream(any(), any())).thenReturn(new InputStreamWrapper(inputStream, 5L));
    doNothing().when(jobStore).removeData(any(), anyString());
    when(executor.getCachedValue(eq("id1"))).thenReturn("/root/Album 1");
    UUID jobId = UUID.randomUUID();
    Collection<PhotoAlbum> albums = ImmutableList.of(new PhotoAlbum("id1", "Album 1", "This is a fake album"));
    Collection<PhotoModel> photos = ImmutableList.of(new PhotoModel("pic1.jpg", "http://fake.com/1.jpg", "A pic", "image/jpeg", "p1", "id1", true), new PhotoModel("pic2.png", "https://fake.com/2.png", "fine art", "image/png", "p2", "id1", true));
    PhotosContainerResource resource = spy(new PhotosContainerResource(albums, photos));
    importer.importItem(jobId, executor, authData, resource);
    InOrder clientInOrder = Mockito.inOrder(client);
    verify(resource).transmogrify(any(KoofrTransmogrificationConfig.class));
    clientInOrder.verify(client).ensureRootFolder();
    clientInOrder.verify(client).ensureFolder("/root", "Album 1");
    clientInOrder.verify(client).uploadFile(eq("/root/Album 1"), eq("pic1.jpg"), any(), eq("image/jpeg"), isNull(), eq("A pic"));
    clientInOrder.verify(client).uploadFile(eq("/root/Album 1"), eq("pic2.png"), any(), eq("image/png"), isNull(), eq("fine art"));
    verify(jobStore, Mockito.times(2)).removeData(any(), anyString());
}

Example 3 with InputStreamWrapper

use of org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper in project data-transfer-project by google.

the class GoogleTempFileStore method getStream.

InputStreamWrapper getStream(UUID jobId, String keyName) {
    String blobName = getDataKeyName(jobId, keyName);
    Blob blob = bucket.get(blobName);
    ReadChannel channel = blob.reader();
    return new InputStreamWrapper(Channels.newInputStream(channel), blob.getSize());
}

Example 4 with InputStreamWrapper

use of org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper in project data-transfer-project by google.

the class MicrosoftPhotosImporterTest method testImportItemAllSuccess.

@Test
public void testImportItemAllSuccess() throws Exception {
    List<PhotoAlbum> albums = ImmutableList.of(new PhotoAlbum("id1", "albumb1", "This is a fake albumb"));
    List<PhotoModel> photos = ImmutableList.of(new PhotoModel("Pic1", "http://fake.com/1.jpg", "A pic", "image/jpg", "p1", "id1", true), new PhotoModel("Pic2", "https://fake.com/2.png", "fine art", "image/png", "p2", "id1", true));
    when(jobStore.getStream(uuid, "http://fake.com/1.jpg")).thenReturn(new InputStreamWrapper(new ByteArrayInputStream(new byte[CHUNK_SIZE])));
    when(jobStore.getStream(uuid, "https://fake.com/2.png")).thenReturn(new InputStreamWrapper(new ByteArrayInputStream(new byte[CHUNK_SIZE])));
    PhotosContainerResource data = new PhotosContainerResource(albums, photos);
    Call call = mock(Call.class);
    doReturn(call).when(client).newCall(argThat((Request r) -> r.url().toString().equals("https://www.baseurl.com/v1.0/me/drive/special/photos/children")));
    Response response = mock(Response.class);
    ResponseBody body = mock(ResponseBody.class);
    when(body.bytes()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"id1\"}").bytes());
    when(body.string()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"id1\"}").string());
    when(response.code()).thenReturn(200);
    when(response.body()).thenReturn(body);
    when(call.execute()).thenReturn(response);
    Call call2 = mock(Call.class);
    doReturn(call2).when(client).newCall(argThat((Request r) -> r.url().toString().contains("createUploadSession")));
    Response response2 = mock(Response.class);
    ResponseBody body2 = mock(ResponseBody.class);
    when(body2.bytes()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"uploadUrl\": \"https://scalia.com/link\"}").bytes());
    when(body2.string()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"uploadUrl\": \"https://scalia.com/link\"}").string());
    when(response2.code()).thenReturn(200);
    when(response2.body()).thenReturn(body2);
    when(call2.execute()).thenReturn(response2);
    Call call3 = mock(Call.class);
    doReturn(call3).when(client).newCall(argThat((Request r) -> r.url().toString().contains("scalia.com/link")));
    Response response3 = mock(Response.class);
    ResponseBody body3 = mock(ResponseBody.class);
    when(body3.bytes()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"rand1\"}").bytes());
    when(body3.string()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"rand1\"}").string());
    when(response3.code()).thenReturn(200);
    when(response3.body()).thenReturn(body3);
    when(call3.execute()).thenReturn(response3);
    ImportResult result = importer.importItem(uuid, executor, authData, data);
    verify(client, times(5)).newCall(any());
    assertThat(result).isEqualTo(ImportResult.OK);
}

Example 5 with InputStreamWrapper

use of org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper in project data-transfer-project by google.

the class KoofrPhotosImporterTest method testImportItemFromJobStoreUserTimeZoneCalledOnce.

@Test
public void testImportItemFromJobStoreUserTimeZoneCalledOnce() throws Exception {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] { 0, 1, 2, 3, 4 });
    when(jobStore.getStream(any(), any())).thenReturn(new InputStreamWrapper(inputStream, 5L));
    UUID jobId = UUID.randomUUID();
    PortabilityJob job = mock(PortabilityJob.class);
    when(job.userTimeZone()).thenReturn(TimeZone.getTimeZone("Europe/Rome"));
    when(jobStore.findJob(jobId)).thenReturn(job);
    Collection<PhotoAlbum> albums = ImmutableList.of(new PhotoAlbum("id1", "Album 1", "This is a fake album"));
    DateFormat format = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss");
    format.setTimeZone(TimeZone.getTimeZone("Europe/Kiev"));
    Collection<PhotoModel> photos1 = ImmutableList.of(new PhotoModel("pic1.jpg", "http://fake.com/1.jpg", "A pic", "image/jpeg", "p1", "id1", true, format.parse("2021:02:16 11:55:00")));
    Collection<PhotoModel> photos2 = ImmutableList.of(new PhotoModel("pic2.jpg", "http://fake.com/2.jpg", "A pic", "image/jpeg", "p2", "id1", true, format.parse("2021:02:17 11:55:00")));
    PhotosContainerResource resource1 = spy(new PhotosContainerResource(albums, photos1));
    PhotosContainerResource resource2 = spy(new PhotosContainerResource(albums, photos2));
    importer.importItem(jobId, executor, authData, resource1);
    importer.importItem(jobId, executor, authData, resource2);
    InOrder clientInOrder = Mockito.inOrder(client);
    String[] titles = { "2021-02-16 10.55.00 pic1.jpg", "2021-02-17 10.55.00 pic2.jpg" };
    for (String title : titles) {
        clientInOrder.verify(client).uploadFile(any(), eq(title), any(), any(), any(), any());
    }
    verify(jobStore, atMostOnce()).findJob(jobId);
}