Examples with BatchMediaItemResponse org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse used on opensource projects

Example 1 with BatchMediaItemResponse

use of org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse in project data-transfer-project by google.

the class GooglePhotosImporterTest method importTwoPhotosWithFailure.

@Test
public void importTwoPhotosWithFailure() throws Exception {
    PhotoModel photoModel1 = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID1", OLD_ALBUM_ID, false);
    PhotoModel photoModel2 = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID2", OLD_ALBUM_ID, false);
    Mockito.when(googlePhotosInterface.uploadPhotoContent(any())).thenReturn("token1", "token2");
    BatchMediaItemResponse batchMediaItemResponse = new BatchMediaItemResponse(new NewMediaItemResult[] { buildMediaItemResult("token1", Code.OK_VALUE), buildMediaItemResult("token2", Code.UNAUTHENTICATED_VALUE) });
    Mockito.when(googlePhotosInterface.createPhotos(any(NewMediaItemUpload.class))).thenReturn(batchMediaItemResponse);
    long length = googlePhotosImporter.importPhotoBatch(UUID.randomUUID(), Mockito.mock(TokensAndUrlAuthData.class), Lists.newArrayList(photoModel1, photoModel2), executor, NEW_ALBUM_ID);
    // Only one photo of 32L imported
    assertEquals(32L, length);
    assertTrue(executor.isKeyCached(String.format("%s-%s", OLD_ALBUM_ID, "oldPhotoID1")));
    String failedDataId = String.format("%s-%s", OLD_ALBUM_ID, "oldPhotoID2");
    assertFalse(executor.isKeyCached(failedDataId));
    ErrorDetail errorDetail = executor.getErrors().iterator().next();
    assertEquals(failedDataId, errorDetail.id());
    assertThat(errorDetail.exception(), CoreMatchers.containsString("Media item could not be created."));
}

Example 2 with BatchMediaItemResponse

use of org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse in project data-transfer-project by google.

the class GooglePhotosImporterTest method importPhotoInTempStore.

@Test
public void importPhotoInTempStore() throws Exception {
    PhotoModel photoModel = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID1", OLD_ALBUM_ID, true);
    Mockito.when(googlePhotosInterface.uploadPhotoContent(any())).thenReturn("token1");
    JobStore jobStore = Mockito.mock(LocalJobStore.class);
    Mockito.when(jobStore.getStream(any(), any())).thenReturn(new TemporaryPerJobDataStore.InputStreamWrapper(new ByteArrayInputStream("TestingBytes".getBytes())));
    Mockito.doNothing().when(jobStore).removeData(any(), anyString());
    GooglePhotosImporter googlePhotosImporter = new GooglePhotosImporter(null, jobStore, null, null, googlePhotosInterface, null, null, 1.0);
    BatchMediaItemResponse batchMediaItemResponse = new BatchMediaItemResponse(new NewMediaItemResult[] { buildMediaItemResult("token1", Code.OK_VALUE) });
    Mockito.when(googlePhotosInterface.createPhotos(any(NewMediaItemUpload.class))).thenReturn(batchMediaItemResponse);
    UUID jobId = UUID.randomUUID();
    long length = googlePhotosImporter.importPhotoBatch(jobId, Mockito.mock(TokensAndUrlAuthData.class), Lists.newArrayList(photoModel), executor, NEW_ALBUM_ID);
    assertTrue(executor.isKeyCached(String.format("%s-%s", OLD_ALBUM_ID, "oldPhotoID1")));
    Mockito.verify(jobStore, Mockito.times(1)).removeData(any(), anyString());
    Mockito.verify(jobStore, Mockito.times(1)).getStream(any(), anyString());
}

Example 3 with BatchMediaItemResponse

use of org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse in project data-transfer-project by google.

the class GooglePhotosImporterTest method importTwoPhotos.

@Test
public void importTwoPhotos() throws Exception {
    PhotoModel photoModel1 = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID1", OLD_ALBUM_ID, false);
    PhotoModel photoModel2 = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID2", OLD_ALBUM_ID, false);
    Mockito.when(googlePhotosInterface.uploadPhotoContent(any())).thenReturn("token1", "token2");
    BatchMediaItemResponse batchMediaItemResponse = new BatchMediaItemResponse(new NewMediaItemResult[] { buildMediaItemResult("token1", Code.OK_VALUE), buildMediaItemResult("token2", Code.OK_VALUE) });
    Mockito.when(googlePhotosInterface.createPhotos(any(NewMediaItemUpload.class))).thenReturn(batchMediaItemResponse);
    long length = googlePhotosImporter.importPhotoBatch(UUID.randomUUID(), Mockito.mock(TokensAndUrlAuthData.class), Lists.newArrayList(photoModel1, photoModel2), executor, NEW_ALBUM_ID);
    // Two photos of 32L each imported
    assertEquals(64L, length);
    assertTrue(executor.isKeyCached(String.format("%s-%s", OLD_ALBUM_ID, "oldPhotoID1")));
    assertTrue(executor.isKeyCached(String.format("%s-%s", OLD_ALBUM_ID, "oldPhotoID2")));
}

Example 4 with BatchMediaItemResponse

use of org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse in project data-transfer-project by google.

the class GooglePhotosImporterTest method importPhotoInTempStoreFailure.

@Test
public void importPhotoInTempStoreFailure() throws Exception {
    PhotoModel photoModel = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID1", OLD_ALBUM_ID, true);
    Mockito.when(googlePhotosInterface.uploadPhotoContent(any())).thenThrow(new IOException("Unit Testing"));
    JobStore jobStore = Mockito.mock(LocalJobStore.class);
    Mockito.when(jobStore.getStream(any(), any())).thenReturn(new TemporaryPerJobDataStore.InputStreamWrapper(new ByteArrayInputStream("TestingBytes".getBytes())));
    Mockito.doNothing().when(jobStore).removeData(any(), anyString());
    GooglePhotosImporter googlePhotosImporter = new GooglePhotosImporter(null, jobStore, null, null, googlePhotosInterface, null, null, 1.0);
    BatchMediaItemResponse batchMediaItemResponse = new BatchMediaItemResponse(new NewMediaItemResult[] { buildMediaItemResult("token1", Code.OK_VALUE) });
    Mockito.when(googlePhotosInterface.createPhotos(any(NewMediaItemUpload.class))).thenReturn(batchMediaItemResponse);
    UUID jobId = UUID.randomUUID();
    googlePhotosImporter.importPhotoBatch(jobId, Mockito.mock(TokensAndUrlAuthData.class), Lists.newArrayList(photoModel), executor, NEW_ALBUM_ID);
    Mockito.verify(jobStore, Mockito.times(0)).removeData(any(), anyString());
    Mockito.verify(jobStore, Mockito.times(1)).getStream(any(), anyString());
}

Example 5 with BatchMediaItemResponse

use of org.datatransferproject.datatransfer.google.mediaModels.BatchMediaItemResponse in project data-transfer-project by google.

the class GooglePhotosImporter method importPhotoBatch.

long importPhotoBatch(UUID jobId, TokensAndUrlAuthData authData, List<PhotoModel> photos, IdempotentImportExecutor executor, String albumId) throws Exception {
    final ArrayList<NewMediaItem> mediaItems = new ArrayList<>();
    final HashMap<String, PhotoModel> uploadTokenToDataId = new HashMap<>();
    final HashMap<String, Long> uploadTokenToLength = new HashMap<>();
    // this however, seems to require knowledge of the total file size.
    for (PhotoModel photo : photos) {
        try {
            Pair<InputStream, Long> inputStreamBytesPair = getInputStreamForUrl(jobId, photo.getFetchableUrl(), photo.isInTempStore());
            try (InputStream s = inputStreamBytesPair.getFirst()) {
                String uploadToken = getOrCreatePhotosInterface(jobId, authData).uploadPhotoContent(s);
                mediaItems.add(new NewMediaItem(cleanDescription(photo.getDescription()), uploadToken));
                uploadTokenToDataId.put(uploadToken, photo);
                uploadTokenToLength.put(uploadToken, inputStreamBytesPair.getSecond());
            }
            try {
                if (photo.isInTempStore()) {
                    jobStore.removeData(jobId, photo.getFetchableUrl());
                }
            } catch (Exception e) {
                // Swallow the exception caused by Remove data so that existing flows continue
                monitor.info(() -> format("%s: Exception swallowed in removeData call for localPath %s", jobId, photo.getFetchableUrl()), e);
            }
        } catch (IOException e) {
            executor.executeAndSwallowIOExceptions(IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), photo.getTitle(), () -> {
                throw e;
            });
        }
    }
    if (mediaItems.isEmpty()) {
        // Either we were not passed in any videos or we failed upload on all of them.
        return 0L;
    }
    long totalBytes = 0L;
    NewMediaItemUpload uploadItem = new NewMediaItemUpload(albumId, mediaItems);
    try {
        BatchMediaItemResponse photoCreationResponse = getOrCreatePhotosInterface(jobId, authData).createPhotos(uploadItem);
        Preconditions.checkNotNull(photoCreationResponse);
        NewMediaItemResult[] mediaItemResults = photoCreationResponse.getResults();
        Preconditions.checkNotNull(mediaItemResults);
        for (NewMediaItemResult mediaItem : mediaItemResults) {
            PhotoModel photo = uploadTokenToDataId.get(mediaItem.getUploadToken());
            totalBytes += processMediaResult(mediaItem, IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), executor, photo.getTitle(), uploadTokenToLength.get(mediaItem.getUploadToken()));
            uploadTokenToDataId.remove(mediaItem.getUploadToken());
        }
        if (!uploadTokenToDataId.isEmpty()) {
            for (PhotoModel photo : uploadTokenToDataId.values()) {
                executor.executeAndSwallowIOExceptions(IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), photo.getTitle(), () -> {
                    throw new IOException("Photo was missing from results list.");
                });
            }
        }
    } catch (IOException e) {
        if (e.getMessage() != null && e.getMessage().contains("The remaining storage in the user's account is not enough")) {
            throw new DestinationMemoryFullException("Google destination storage full", e);
        } else {
            throw e;
        }
    }
    return totalBytes;
}