Examples with ContinuationData org.datatransferproject.spi.transfer.types.ContinuationData used on opensource projects

Example 21 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleMailExporter method export.

@Override
public ExportResult<MailContainerResource> export(UUID id, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation) {
    // Create a new gmail service for the authorized user
    Gmail gmail = getOrCreateGmail(authData);
    Messages.List request = null;
    try {
        request = gmail.users().messages().list(USER).setMaxResults(PAGE_SIZE);
    } catch (IOException e) {
        return new ExportResult<>(e);
    }
    if (exportInformation.isPresent() && exportInformation.get().getPaginationData() != null) {
        request.setPageToken(((StringPaginationToken) exportInformation.get().getPaginationData()).getToken());
    }
    ListMessagesResponse response = null;
    try {
        response = request.execute();
    } catch (IOException e) {
        return new ExportResult<>(e);
    }
    List<MailMessageModel> results = new ArrayList<>(response.getMessages().size());
    // as we can't store all the mail messages in memory at once.
    for (Message listMessage : response.getMessages()) {
        Message getResponse = null;
        try {
            getResponse = gmail.users().messages().get(USER, listMessage.getId()).setFormat("raw").execute();
        } catch (IOException e) {
            return new ExportResult<>(e);
        }
        // TODO: note this doesn't transfer things like labels
        results.add(new MailMessageModel(getResponse.getRaw(), getResponse.getLabelIds()));
    }
    PaginationData newPage = null;
    ResultType resultType = ResultType.END;
    if (response.getNextPageToken() != null) {
        newPage = new StringPaginationToken(response.getNextPageToken());
        resultType = ResultType.CONTINUE;
    }
    MailContainerResource mailContainerResource = new MailContainerResource(null, results);
    return new ExportResult<>(resultType, mailContainerResource, new ContinuationData(newPage));
}

Example 22 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GooglePhotosExporter method exportAlbums.

/**
 * Note: not all accounts have albums to return. In that case, we just return an empty list of
 * albums instead of trying to iterate through a null list.
 */
@VisibleForTesting
ExportResult<PhotosContainerResource> exportAlbums(TokensAndUrlAuthData authData, Optional<PaginationData> paginationData, UUID jobId) throws IOException, InvalidTokenException, PermissionDeniedException {
    Optional<String> paginationToken = Optional.empty();
    if (paginationData.isPresent()) {
        String token = ((StringPaginationToken) paginationData.get()).getToken();
        Preconditions.checkArgument(token.startsWith(ALBUM_TOKEN_PREFIX), "Invalid pagination token " + token);
        paginationToken = Optional.of(token.substring(ALBUM_TOKEN_PREFIX.length()));
    }
    AlbumListResponse albumListResponse;
    albumListResponse = getOrCreatePhotosInterface(authData).listAlbums(paginationToken);
    PaginationData nextPageData;
    String token = albumListResponse.getNextPageToken();
    List<PhotoAlbum> albums = new ArrayList<>();
    GoogleAlbum[] googleAlbums = albumListResponse.getAlbums();
    if (Strings.isNullOrEmpty(token)) {
        nextPageData = new StringPaginationToken(PHOTO_TOKEN_PREFIX);
    } else {
        nextPageData = new StringPaginationToken(ALBUM_TOKEN_PREFIX + token);
    }
    ContinuationData continuationData = new ContinuationData(nextPageData);
    if (googleAlbums != null && googleAlbums.length > 0) {
        for (GoogleAlbum googleAlbum : googleAlbums) {
            // Add album info to list so album can be recreated later
            PhotoAlbum photoAlbum = new PhotoAlbum(googleAlbum.getId(), googleAlbum.getTitle(), null);
            albums.add(photoAlbum);
            monitor.debug(() -> String.format("%s: Google Photos exporting album: %s", jobId, photoAlbum.getId()));
            // Add album id to continuation data
            continuationData.addContainerResource(new IdOnlyContainerResource(googleAlbum.getId()));
        }
    }
    ResultType resultType = ResultType.CONTINUE;
    PhotosContainerResource containerResource = new PhotosContainerResource(albums, null);
    return new ExportResult<>(resultType, containerResource, continuationData);
}

Example 23 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GooglePlusExporter method export.

@Override
public ExportResult<SocialActivityContainerResource> export(UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation) throws IOException {
    Plus plus = getOrCreatePeopleService(authData);
    Plus.Activities.List listActivities = plus.activities().list("me", "public");
    if (exportInformation.isPresent()) {
        StringPaginationToken pageToken = (StringPaginationToken) exportInformation.get().getPaginationData();
        listActivities.setPageToken(pageToken.getToken());
    }
    ActivityFeed activityFeed = listActivities.execute();
    List<Activity> activities = activityFeed.getItems();
    ContinuationData continuationData = null;
    SocialActivityContainerResource results = null;
    if (activities != null && !activities.isEmpty()) {
        List<SocialActivityModel> activityModels = new ArrayList<>();
        Activity.Actor actor = activities.get(0).getActor();
        SocialActivityActor parsedActor = new SocialActivityActor(actor.getUrl(), actor.getDisplayName(), actor.getUrl());
        if (!Strings.isNullOrEmpty(activityFeed.getNextPageToken())) {
            continuationData = new ContinuationData(new StringPaginationToken(activityFeed.getNextPageToken()));
        }
        for (Activity activity : activities) {
            try {
                activityModels.add(postToActivityModel(activity));
            } catch (RuntimeException e) {
                throw new IOException("Problem exporting: " + activity, e);
            }
        }
        results = new SocialActivityContainerResource(jobId.toString(), parsedActor, activityModels);
    }
    return new ExportResult<>(continuationData == null ? ResultType.END : ResultType.CONTINUE, results, continuationData);
}

Example 24 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleMailExporterTest method exportMessagesSubsequentSet.

@Test
public void exportMessagesSubsequentSet() throws IOException {
    setUpSingleMessageResponse();
    // Looking at subsequent page, with no page after it
    PaginationData paginationData = new StringPaginationToken(NEXT_TOKEN);
    ExportInformation exportInformation = new ExportInformation(paginationData, null);
    messageListResponse.setNextPageToken(null);
    // Run test
    ExportResult<MailContainerResource> result = googleMailExporter.export(JOB_ID, null, Optional.of(exportInformation));
    // Check results
    // Verify correct calls were made (i.e., token was set before execution)
    InOrder inOrder = Mockito.inOrder(messageListRequest);
    inOrder.verify(messageListRequest).setPageToken(NEXT_TOKEN);
    inOrder.verify(messageListRequest).execute();
    // Check pagination token (should be null)
    ContinuationData continuationData = (ContinuationData) result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken).isNull();
}

Example 25 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GooglePhotosExporterTest method exportAlbumFirstSet.

@Test
public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException {
    setUpSingleAlbum();
    when(albumListResponse.getNextPageToken()).thenReturn(ALBUM_TOKEN);
    // Run test
    ExportResult<PhotosContainerResource> result = googlePhotosExporter.exportAlbums(null, Optional.empty(), uuid);
    // Check results
    // Verify correct methods were called
    verify(photosInterface).listAlbums(Optional.empty());
    verify(albumListResponse).getAlbums();
    // Check pagination token
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(ALBUM_TOKEN_PREFIX + ALBUM_TOKEN);
    // Check albums field of container
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums.stream().map(PhotoAlbum::getId).collect(Collectors.toList())).containsExactly(ALBUM_ID);
    // Check photos field of container (should be empty, even though there is a photo in the
    // original album)
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos).isEmpty();
    // Should be one container in the resource list
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources.stream().map(a -> ((IdOnlyContainerResource) a).getId()).collect(Collectors.toList())).containsExactly(ALBUM_ID);
}