Examples with ContinuationData org.datatransferproject.spi.transfer.types.ContinuationData used on opensource projects

Example 26 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class MicrosoftContactsExporter method doExport.

@SuppressWarnings("unchecked")
private ExportResult<ContactsModelWrapper> doExport(TokensAndUrlAuthData authData, String url) {
    Request.Builder graphReqBuilder = new Request.Builder().url(url);
    graphReqBuilder.header("Authorization", "Bearer " + authData.getAccessToken());
    try (Response graphResponse = client.newCall(graphReqBuilder.build()).execute()) {
        ResponseBody body = graphResponse.body();
        if (body == null) {
            return new ExportResult<>(new Exception("Error retrieving contacts: response body was null"));
        }
        String graphBody = new String(body.bytes());
        Map graphMap = objectMapper.reader().forType(Map.class).readValue(graphBody);
        String nextLink = (String) graphMap.get(ODATA_NEXT);
        ContinuationData continuationData = nextLink == null ? null : new ContinuationData(new GraphPagination(nextLink));
        List<Map<String, Object>> rawContacts = (List<Map<String, Object>>) graphMap.get("value");
        if (rawContacts == null) {
            return new ExportResult<>(ExportResult.ResultType.END);
        }
        ContactsModelWrapper wrapper = transform(rawContacts);
        return new ExportResult<>(ExportResult.ResultType.CONTINUE, wrapper, continuationData);
    } catch (IOException e) {
        // FIXME log error
        e.printStackTrace();
        return new ExportResult<>(e);
    }
}

Example 27 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class MastodonActivityExport method export.

@Override
public ExportResult<SocialActivityContainerResource> export(UUID jobId, CookiesAndUrlAuthData authData, Optional<ExportInformation> exportInformation) throws Exception {
    checkState(authData.getCookies().size() == 1, "Exactly 1 cookie expected: %s", authData.getCookies());
    String maxId = null;
    if (exportInformation.isPresent()) {
        StringPaginationToken pageData = (StringPaginationToken) exportInformation.get().getPaginationData();
        if (!Strings.isNullOrEmpty(pageData.getToken())) {
            maxId = pageData.getToken();
        }
    }
    MastodonHttpUtilities utilities = new MastodonHttpUtilities(authData.getCookies().get(0), authData.getUrl());
    Account account = utilities.getAccount();
    Status[] statuses = utilities.getStatuses(maxId);
    List<SocialActivityModel> activityList = new ArrayList<>(statuses.length);
    SocialActivityActor actor = new SocialActivityActor("acct:" + account.getUsername() + "@" + utilities.getHostName(), account.getDisplayName(), account.getUrl());
    ContinuationData continuationData = null;
    if (statuses.length > 0) {
        String lastId = null;
        for (Status status : statuses) {
            activityList.add(statusToActivity(account, status, utilities));
            lastId = status.getId();
        }
        continuationData = new ContinuationData(new StringPaginationToken(lastId));
    }
    return new ExportResult<>(continuationData == null ? ResultType.END : ResultType.CONTINUE, new SocialActivityContainerResource(account.getId() + maxId, actor, activityList), continuationData);
}

Example 28 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleVideosExporterTest method exportSingleVideo.

@Test
public void exportSingleVideo() throws IOException {
    when(albumListResponse.getNextPageToken()).thenReturn(null);
    GoogleMediaItem mediaItem = setUpSingleVideo(VIDEO_URI, VIDEO_ID);
    when(mediaItemSearchResponse.getMediaItems()).thenReturn(new GoogleMediaItem[] { mediaItem });
    when(mediaItemSearchResponse.getNextPageToken()).thenReturn(VIDEO_TOKEN);
    // Run test
    ExportResult<VideosContainerResource> result = googleVideosExporter.exportVideos(null, Optional.empty());
    // Verify correct methods were called
    verify(videosInterface).listVideoItems(Optional.empty());
    verify(mediaItemSearchResponse).getMediaItems();
    // Check pagination
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(VIDEO_TOKEN);
    // Check videos field of container
    Collection<VideoModel> actualVideos = result.getExportedData().getVideos();
    URI video_uri_object = null;
    try {
        video_uri_object = new URI(VIDEO_URI + "=dv");
    } catch (URISyntaxException e) {
        e.printStackTrace();
    }
    assertThat(actualVideos.stream().map(VideoModel::getContentUrl).collect(Collectors.toList())).containsExactly(video_uri_object);
    // Since albums are not supported atm, this should be null
    assertThat(actualVideos.stream().map(VideoModel::getAlbumId).collect(Collectors.toList())).containsExactly((Object) null);
}

Example 29 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class ImgurPhotosExporter method requestAlbums.

/**
 * Exports albums.
 *
 * @param authData authentication information
 * @param paginationData pagination information to use for subsequent calls
 */
private ExportResult<PhotosContainerResource> requestAlbums(TokensAndUrlAuthData authData, PaginationData paginationData) throws IOException {
    ImmutableList.Builder<PhotoAlbum> albumBuilder = ImmutableList.builder();
    List<IdOnlyContainerResource> albumIds = new ArrayList<>();
    int page = paginationData == null ? 0 : ((IntPaginationToken) paginationData).getStart();
    String url = format(ALBUMS_URL_TEMPLATE, page);
    List<Map<String, Object>> items = requestData(authData, url);
    // Request result doesn't indicate if it's the last page
    boolean hasMore = (items != null && items.size() != 0);
    for (Map<String, Object> item : items) {
        albumBuilder.add(new PhotoAlbum((String) item.get("id"), (String) item.get("title"), (String) item.get("description")));
        // Save album id for recalling export to get all the photos in albums
        albumIds.add(new IdOnlyContainerResource((String) item.get("id")));
    }
    if (page == 0) {
        // For checking non-album photos. Their export should be performed after all the others
        // Album will be created later
        albumIds.add(new IdOnlyContainerResource(DEFAULT_ALBUM_ID));
    }
    PaginationData newPage = null;
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
        int start = ((IntPaginationToken) newPage).getStart();
        monitor.info(() -> format("albums size: %s, newPage: %s", items.size(), start));
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albumBuilder.build(), null);
    ContinuationData continuationData = new ContinuationData(newPage);
    albumIds.forEach(continuationData::addContainerResource);
    ExportResult.ResultType resultType = ExportResult.ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ExportResult.ResultType.END;
    }
    return new ExportResult<>(resultType, photosContainerResource, continuationData);
}

Example 30 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class MicrosoftPhotosExporterTest method exportAlbumAndPhotoWithNextPage.

@Test
public void exportAlbumAndPhotoWithNextPage() throws IOException {
    // Setup
    MicrosoftDriveItem folderItem = setUpSingleAlbum();
    MicrosoftDriveItem photoItem = setUpSinglePhoto(IMAGE_URI, PHOTO_ID);
    when(driveItemsResponse.getDriveItems()).thenReturn(new MicrosoftDriveItem[] { folderItem, photoItem });
    when(driveItemsResponse.getNextPageLink()).thenReturn(DRIVE_PAGE_URL);
    // Run
    ExportResult<PhotosContainerResource> result = microsoftPhotosExporter.exportOneDrivePhotos(null, Optional.empty(), Optional.empty(), uuid);
    // Verify method calls
    verify(photosInterface).getDriveItemsFromSpecialFolder(MicrosoftSpecialFolder.FolderType.photos);
    verify(driveItemsResponse).getDriveItems();
    // Verify pagination token is set
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(DRIVE_TOKEN_PREFIX + DRIVE_PAGE_URL);
    // Verify one album is ready for import
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums.stream().map(PhotoAlbum::getId).collect(Collectors.toList())).containsExactly(FOLDER_ID);
    // Verify one photo should be present (in the root Photos special folder)
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos.stream().map(PhotoModel::getFetchableUrl).collect(Collectors.toList())).containsExactly(IMAGE_URI);
    assertThat(actualPhotos.stream().map(PhotoModel::getAlbumId).collect(Collectors.toList())).containsExactly(null);
    assertThat(actualPhotos.stream().map(PhotoModel::getTitle).collect(Collectors.toList())).containsExactly(FILENAME);
    // Verify there is one container ready for sub-processing
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources.stream().map(a -> ((IdOnlyContainerResource) a).getId()).collect(Collectors.toList())).containsExactly(FOLDER_ID);
}