Examples with DestinationMemoryFullException org.datatransferproject.spi.transfer.types.DestinationMemoryFullException used on opensource projects

Example 1 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class FlickrPhotosImporter method importItem.

@Override
public ImportResult importItem(UUID jobId, IdempotentImportExecutor idempotentExecutor, AuthData authData, PhotosContainerResource data) throws Exception, IOException {
    Auth auth;
    try {
        auth = FlickrUtils.getAuth(authData, flickr);
    } catch (FlickrException e) {
        return new ImportResult(e);
    }
    RequestContext.getRequestContext().setAuth(auth);
    Preconditions.checkArgument(data.getAlbums() != null || data.getPhotos() != null, "Error: There is no data to import");
    if (data.getAlbums() != null) {
        storeAlbums(jobId, data.getAlbums());
    }
    if (data.getPhotos() != null) {
        for (PhotoModel photo : data.getPhotos()) {
            try {
                importSinglePhoto(idempotentExecutor, jobId, photo);
            } catch (FlickrException e) {
                if (e.getMessage().contains("Upload limit reached")) {
                    throw new DestinationMemoryFullException("Flickr destination memory reached", e);
                } else if (e.getMessage().contains("Photo already in set")) {
                    // uploaded
                    continue;
                }
                throw new IOException(e);
            }
        }
    }
    return new ImportResult(ImportResult.ResultType.OK);
}

Example 2 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class KoofrPhotosImporter method importSinglePhoto.

private String importSinglePhoto(PhotoModel photo, UUID jobId, IdempotentImportExecutor idempotentImportExecutor, KoofrClient koofrClient) throws IOException, InvalidTokenException, DestinationMemoryFullException {
    monitor.debug(() -> String.format("Import single photo %s", photo.getTitle()));
    BufferedInputStream inputStream = null;
    try {
        if (photo.isInTempStore()) {
            inputStream = new BufferedInputStream(jobStore.getStream(jobId, photo.getFetchableUrl()).getStream());
        } else if (photo.getFetchableUrl() != null) {
            HttpURLConnection conn = imageStreamProvider.getConnection(photo.getFetchableUrl());
            inputStream = new BufferedInputStream(conn.getInputStream());
        } else {
            throw new IllegalStateException("Don't know how to get the inputStream for " + photo.getTitle());
        }
        final byte[] bytes = IOUtils.toByteArray(inputStream);
        Date dateCreated = getDateCreated(photo, bytes);
        String title = buildPhotoTitle(jobId, photo.getTitle(), dateCreated);
        String description = KoofrClient.trimDescription(photo.getDescription());
        String parentPath = idempotentImportExecutor.getCachedValue(photo.getAlbumId());
        String fullPath = parentPath + "/" + title;
        if (koofrClient.fileExists(fullPath)) {
            monitor.debug(() -> String.format("Photo already exists %s", photo.getTitle()));
            return fullPath;
        }
        final ByteArrayInputStream inMemoryInputStream = new ByteArrayInputStream(bytes);
        String response = koofrClient.uploadFile(parentPath, title, inMemoryInputStream, photo.getMediaType(), dateCreated, description);
        try {
            if (photo.isInTempStore()) {
                jobStore.removeData(jobId, photo.getFetchableUrl());
            }
        } catch (Exception e) {
            // Swallow the exception caused by Remove data so that existing flows continue
            monitor.info(() -> format("Exception swallowed while removing data for jobId %s, localPath %s", jobId, photo.getFetchableUrl()), e);
        }
        return response;
    } finally {
        if (inputStream != null) {
            inputStream.close();
        }
    }
}

Example 3 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class KoofrClientTest method testUploadFileOutOfSpace.

@Test
public void testUploadFileOutOfSpace() throws Exception {
    server.enqueue(new MockResponse().setResponseCode(413).setHeader("Content-Type", "application/json").setBody("{\"error\":{\"code\":\"QuotaExceeded\",\"message\":\"Quota exceeded\"},\"requestId\":\"bad2465e-300e-4079-57ad-46b256e74d21\"}"));
    final InputStream inputStream = new ByteArrayInputStream(new byte[] { 0, 1, 2, 3, 4 });
    DestinationMemoryFullException caughtExc = null;
    try {
        client.uploadFile("/path/to/folder", "image.jpg", inputStream, "image/jpeg", null, null);
    } catch (DestinationMemoryFullException exc) {
        caughtExc = exc;
    }
    Assert.assertNotNull(caughtExc);
    Assert.assertEquals("Koofr quota exceeded", caughtExc.getMessage());
    Assert.assertEquals(1, server.getRequestCount());
    final RecordedRequest recordedRequest = server.takeRequest();
    Assert.assertEquals("POST", recordedRequest.getMethod());
    Assert.assertEquals("/content/api/v2/mounts/primary/files/put?path=%2Fpath%2Fto%2Ffolder&filename=image.jpg&autorename=true&info=true", recordedRequest.getPath());
    Assert.assertEquals("Bearer acc", recordedRequest.getHeader("Authorization"));
    Assert.assertEquals("2.1", recordedRequest.getHeader("X-Koofr-Version"));
    Assert.assertEquals("image/jpeg", recordedRequest.getHeader("Content-Type"));
    Assert.assertEquals(5, recordedRequest.getBodySize());
}

Example 4 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class KoofrClient method uploadFile.

@SuppressWarnings("unchecked")
public String uploadFile(String parentPath, String name, InputStream inputStream, String mediaType, Date modified, String description) throws IOException, InvalidTokenException, DestinationMemoryFullException {
    String url;
    try {
        URIBuilder builder = getUriBuilder().setPath(CONTENT_API_PATH_PREFIX + "/mounts/primary/files/put").setParameter("path", parentPath).setParameter("filename", name).setParameter("autorename", "true").setParameter("info", "true");
        if (description != null && description.length() > 0) {
            builder.setParameter("tags", "description=" + description);
        }
        if (modified != null) {
            builder.setParameter("modified", Long.toString(modified.getTime()));
        }
        url = builder.build().toString();
    } catch (URISyntaxException e) {
        throw new IllegalStateException("Could not produce url.", e);
    }
    Request.Builder requestBuilder = getRequestBuilder(url);
    RequestBody uploadBody = new InputStreamRequestBody(MediaType.parse(mediaType), inputStream);
    requestBuilder.post(uploadBody);
    // We need to reset the input stream because the request could already read some data
    try (Response response = getResponse(fileUploadClient, requestBuilder, () -> inputStream.reset())) {
        int code = response.code();
        ResponseBody body = response.body();
        if (code == 413) {
            throw new DestinationMemoryFullException("Koofr quota exceeded", new Exception("Koofr file upload response code " + code));
        }
        if (code < 200 || code > 299) {
            throw new IOException("Got error code: " + code + " message: " + response.message() + " body: " + body.string());
        }
        Map<String, Object> responseData = objectMapper.readValue(body.bytes(), Map.class);
        String newName = (String) responseData.get("name");
        Preconditions.checkState(!Strings.isNullOrEmpty(newName), "Expected name value to be present in %s", responseData);
        return parentPath + "/" + newName;
    }
}

Example 5 with DestinationMemoryFullException

use of org.datatransferproject.spi.transfer.types.DestinationMemoryFullException in project data-transfer-project by google.

the class MicrosoftPhotosImporter method uploadChunk.

// Uploads a single DataChunk to an upload URL
// PUT to {photoUploadUrl}
// HEADERS
// Content-Length: {chunk size in bytes}
// Content-Range: bytes {begin}-{end}/{total size}
// body={bytes}
private Response uploadChunk(DataChunk chunk, String photoUploadUrl, int totalFileSize, String mediaType) throws IOException, DestinationMemoryFullException {
    Request.Builder uploadRequestBuilder = new Request.Builder().url(photoUploadUrl);
    uploadRequestBuilder.header("Authorization", "Bearer " + credential.getAccessToken());
    // put chunk data in
    RequestBody uploadChunkBody = RequestBody.create(MediaType.parse(mediaType), chunk.getData(), 0, chunk.getSize());
    uploadRequestBuilder.put(uploadChunkBody);
    // set chunk data headers, indicating size and chunk range
    final String contentRange = String.format("bytes %d-%d/%d", chunk.getStart(), chunk.getEnd(), totalFileSize);
    uploadRequestBuilder.header("Content-Range", contentRange);
    uploadRequestBuilder.header("Content-Length", String.format("%d", chunk.getSize()));
    // upload the chunk
    Response chunkResponse = client.newCall(uploadRequestBuilder.build()).execute();
    Preconditions.checkNotNull(chunkResponse, "chunkResponse is null");
    if (chunkResponse.code() == 401) {
        // If there was an unauthorized error, then try refreshing the creds
        credentialFactory.refreshCredential(credential);
        monitor.info(() -> "Refreshed authorization token successfuly");
        // update auth info, reupload chunk
        uploadRequestBuilder.header("Authorization", "Bearer " + credential.getAccessToken());
        chunkResponse = client.newCall(uploadRequestBuilder.build()).execute();
    }
    int chunkCode = chunkResponse.code();
    if (chunkCode == 507 && chunkResponse.message().contains("Insufficient Storage")) {
        throw new DestinationMemoryFullException("Microsoft destination storage limit reached", new IOException(String.format("Got error code %d  with message: %s", chunkCode, chunkResponse.message())));
    } else if (chunkCode < 200 || chunkCode > 299) {
        throw new IOException("Got error code: " + chunkCode + " message: " + chunkResponse.message() + " body: " + chunkResponse.body().string());
    } else if (chunkCode == 200 || chunkCode == 201 || chunkCode == 202) {
        monitor.info(() -> String.format("Uploaded chunk %s-%s successfuly, code %d", chunk.getStart(), chunk.getEnd(), chunkCode));
    }
    return chunkResponse;
}