Examples with DestinationMemoryFullException org.datatransferproject.spi.transfer.types.DestinationMemoryFullException used on opensource projects

Example 6 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class MicrosoftPhotosImporter method createUploadSession.

// Request an upload session to the OneDrive api so that we can upload chunks
// to the returned URL
private String createUploadSession(PhotoModel photo, IdempotentImportExecutor idempotentImportExecutor) throws IOException, CopyExceptionWithFailureReason {
    // Forming the URL to create an upload session
    String createSessionUrl;
    if (Strings.isNullOrEmpty(photo.getAlbumId())) {
        createSessionUrl = String.format(albumlessPhotoUrlTemplate, photo.getTitle(), UPLOAD_PARAMS);
    } else {
        String oneDriveFolderId = idempotentImportExecutor.getCachedValue(photo.getAlbumId());
        createSessionUrl = String.format(uploadPhotoUrlTemplate, oneDriveFolderId, photo.getTitle(), UPLOAD_PARAMS);
    }
    // create upload session
    // POST to /me/drive/items/{folder_id}:/{file_name}:/createUploadSession OR /me/drive/items/root:/Photos/{file_name}:/createUploadSession
    // get {uploadurl} from response
    Request.Builder createSessionRequestBuilder = new Request.Builder().url(createSessionUrl);
    // Auth headers
    createSessionRequestBuilder.header("Authorization", "Bearer " + credential.getAccessToken());
    createSessionRequestBuilder.header("Content-Type", "application/json");
    // Post request with empty body. If you don't include an empty body, you'll have problems
    createSessionRequestBuilder.post(RequestBody.create(MediaType.parse("application/json"), objectMapper.writeValueAsString(ImmutableMap.of())));
    // Make the call, we should get an upload url for photo data in a 200 response
    Response response = client.newCall(createSessionRequestBuilder.build()).execute();
    int code = response.code();
    ResponseBody responseBody = response.body();
    // If there was an unauthorized error, then try refreshing the creds
    if (code == 401) {
        this.credentialFactory.refreshCredential(credential);
        monitor.info(() -> "Refreshed authorization token successfuly");
        createSessionRequestBuilder.header("Authorization", "Bearer " + credential.getAccessToken());
        Response newResponse = client.newCall(createSessionRequestBuilder.build()).execute();
        code = newResponse.code();
        responseBody = newResponse.body();
    }
    if (code == 403 && response.message().contains("Access Denied")) {
        throw new PermissionDeniedException("User access to Microsoft One Drive was denied", new IOException(String.format("Got error code %d  with message: %s", code, response.message())));
    } else if (code == 507 && response.message().contains("Insufficient Storage")) {
        throw new DestinationMemoryFullException("Microsoft destination storage limit reached", new IOException(String.format("Got error code %d  with message: %s", code, response.message())));
    } else if (code < 200 || code > 299) {
        throw new IOException(String.format("Got error code: %s\n" + "message: %s\n" + "body: %s\n" + "request url: %s\n" + "bearer token: %s\n" + // For debugging 404s on upload
        " photo: %s\n", code, response.message(), response.body().string(), createSessionUrl, credential.getAccessToken(), photo));
    } else if (code != 200) {
        monitor.info(() -> String.format("Got an unexpected non-200, non-error response code"));
    }
    // make sure we have a non-null response body
    Preconditions.checkState(responseBody != null, "Got Null Body when creating photo upload session %s", photo);
    // convert to a map
    final Map<String, Object> responseData = objectMapper.readValue(responseBody.bytes(), Map.class);
    // return the session's upload url
    Preconditions.checkState(responseData.containsKey("uploadUrl"), "No uploadUrl :(");
    return (String) responseData.get("uploadUrl");
}

Example 7 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class GoogleVideosImporter method importVideoBatch.

long importVideoBatch(List<VideoModel> batchedVideos, PhotosLibraryClient client, IdempotentImportExecutor executor) throws Exception {
    final ArrayList<NewMediaItem> mediaItems = new ArrayList<>();
    final HashMap<String, VideoModel> uploadTokenToDataId = new HashMap<>();
    final HashMap<String, Long> uploadTokenToLength = new HashMap<>();
    // calls of the client to handle the InvalidArgumentException when the user's storage is full.
    try {
        for (VideoModel video : batchedVideos) {
            try {
                Pair<String, Long> pair = uploadMediaItem(video, client);
                final String uploadToken = pair.getLeft();
                mediaItems.add(buildMediaItem(video, uploadToken));
                uploadTokenToDataId.put(uploadToken, video);
                uploadTokenToLength.put(uploadToken, pair.getRight());
            } catch (IOException e) {
                if (e instanceof FileNotFoundException) {
                    // If the video file is no longer available then skip the video. We see this in a small
                    // number of videos where the video has been deleted.
                    monitor.info(() -> String.format("Video resource was missing for id: %s", video.getDataId()), e);
                    continue;
                }
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw e;
                });
            }
        }
        if (mediaItems.isEmpty()) {
            // Either we were not passed in any videos or we failed upload on all of them.
            return 0L;
        }
        BatchCreateMediaItemsResponse response = client.batchCreateMediaItems(mediaItems);
        final List<NewMediaItemResult> resultsList = response.getNewMediaItemResultsList();
        long bytes = 0L;
        for (NewMediaItemResult result : resultsList) {
            String uploadToken = result.getUploadToken();
            Status status = result.getStatus();
            final VideoModel video = uploadTokenToDataId.get(uploadToken);
            Preconditions.checkNotNull(video);
            final int code = status.getCode();
            if (code == Code.OK_VALUE) {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> result.getMediaItem().getId());
                Long length = uploadTokenToLength.get(uploadToken);
                if (length != null) {
                    bytes += length;
                }
            } else {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw new IOException(String.format("Video item could not be created. Code: %d Message: %s", code, result.getStatus().getMessage()));
                });
            }
            uploadTokenToDataId.remove(uploadToken);
        }
        if (!uploadTokenToDataId.isEmpty()) {
            for (VideoModel video : uploadTokenToDataId.values()) {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw new IOException("Video item was missing from results list.");
                });
            }
        }
        return bytes;
    } catch (InvalidArgumentException e) {
        if (e.getMessage().contains("The remaining storage in the user's account is not enough")) {
            throw new DestinationMemoryFullException("Google destination storage full", e);
        } else {
            throw e;
        }
    }
}

Example 8 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class GooglePhotosImporter method importPhotoBatch.

long importPhotoBatch(UUID jobId, TokensAndUrlAuthData authData, List<PhotoModel> photos, IdempotentImportExecutor executor, String albumId) throws Exception {
    final ArrayList<NewMediaItem> mediaItems = new ArrayList<>();
    final HashMap<String, PhotoModel> uploadTokenToDataId = new HashMap<>();
    final HashMap<String, Long> uploadTokenToLength = new HashMap<>();
    // this however, seems to require knowledge of the total file size.
    for (PhotoModel photo : photos) {
        try {
            Pair<InputStream, Long> inputStreamBytesPair = getInputStreamForUrl(jobId, photo.getFetchableUrl(), photo.isInTempStore());
            try (InputStream s = inputStreamBytesPair.getFirst()) {
                String uploadToken = getOrCreatePhotosInterface(jobId, authData).uploadPhotoContent(s);
                mediaItems.add(new NewMediaItem(cleanDescription(photo.getDescription()), uploadToken));
                uploadTokenToDataId.put(uploadToken, photo);
                uploadTokenToLength.put(uploadToken, inputStreamBytesPair.getSecond());
            }
            try {
                if (photo.isInTempStore()) {
                    jobStore.removeData(jobId, photo.getFetchableUrl());
                }
            } catch (Exception e) {
                // Swallow the exception caused by Remove data so that existing flows continue
                monitor.info(() -> format("%s: Exception swallowed in removeData call for localPath %s", jobId, photo.getFetchableUrl()), e);
            }
        } catch (IOException e) {
            executor.executeAndSwallowIOExceptions(IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), photo.getTitle(), () -> {
                throw e;
            });
        }
    }
    if (mediaItems.isEmpty()) {
        // Either we were not passed in any videos or we failed upload on all of them.
        return 0L;
    }
    long totalBytes = 0L;
    NewMediaItemUpload uploadItem = new NewMediaItemUpload(albumId, mediaItems);
    try {
        BatchMediaItemResponse photoCreationResponse = getOrCreatePhotosInterface(jobId, authData).createPhotos(uploadItem);
        Preconditions.checkNotNull(photoCreationResponse);
        NewMediaItemResult[] mediaItemResults = photoCreationResponse.getResults();
        Preconditions.checkNotNull(mediaItemResults);
        for (NewMediaItemResult mediaItem : mediaItemResults) {
            PhotoModel photo = uploadTokenToDataId.get(mediaItem.getUploadToken());
            totalBytes += processMediaResult(mediaItem, IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), executor, photo.getTitle(), uploadTokenToLength.get(mediaItem.getUploadToken()));
            uploadTokenToDataId.remove(mediaItem.getUploadToken());
        }
        if (!uploadTokenToDataId.isEmpty()) {
            for (PhotoModel photo : uploadTokenToDataId.values()) {
                executor.executeAndSwallowIOExceptions(IdempotentImportExecutorHelper.getPhotoIdempotentId(photo), photo.getTitle(), () -> {
                    throw new IOException("Photo was missing from results list.");
                });
            }
        }
    } catch (IOException e) {
        if (e.getMessage() != null && e.getMessage().contains("The remaining storage in the user's account is not enough")) {
            throw new DestinationMemoryFullException("Google destination storage full", e);
        } else {
            throw e;
        }
    }
    return totalBytes;
}