Examples with IntPaginationToken org.datatransferproject.types.common.IntPaginationToken used on opensource projects

Example 1 with IntPaginationToken

use of org.datatransferproject.types.common.IntPaginationToken in project data-transfer-project by google.

the class FlickrPhotosExporter method getPhotos.

private ExportResult<PhotosContainerResource> getPhotos(IdOnlyContainerResource resource, PaginationData paginationData) {
    String photoSetId = resource.getId();
    int page = paginationData == null ? 1 : ((IntPaginationToken) paginationData).getStart();
    PhotoList<Photo> photoSetList;
    try {
        if (Strings.isNullOrEmpty(photoSetId)) {
            RequestContext.getRequestContext().setExtras(EXTRAS);
            perUserRateLimiter.acquire();
            photoSetList = photosInterface.getNotInSet(PHOTO_PER_PAGE, page);
            RequestContext.getRequestContext().setExtras(ImmutableList.of());
        } else {
            perUserRateLimiter.acquire();
            photoSetList = photosetsInterface.getPhotos(photoSetId, ImmutableSet.copyOf(EXTRAS), 0, PHOTO_PER_PAGE, page);
        }
    } catch (FlickrException e) {
        return new ExportResult<>(e);
    }
    boolean hasMore = photoSetList.getPage() != photoSetList.getPages() && !photoSetList.isEmpty();
    Collection<PhotoModel> photos = photoSetList.stream().map(p -> toCommonPhoto(p, photoSetId)).collect(Collectors.toList());
    PaginationData newPage = null;
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
    }
    // Get result type
    ResultType resultType = ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ResultType.END;
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(null, photos);
    return new ExportResult<>(resultType, photosContainerResource, new ContinuationData(newPage));
}

Example 2 with IntPaginationToken

use of org.datatransferproject.types.common.IntPaginationToken in project data-transfer-project by google.

the class ImgurPhotosExporter method requestAlbums.

/**
 * Exports albums.
 *
 * @param authData authentication information
 * @param paginationData pagination information to use for subsequent calls
 */
private ExportResult<PhotosContainerResource> requestAlbums(TokensAndUrlAuthData authData, PaginationData paginationData) throws IOException {
    ImmutableList.Builder<PhotoAlbum> albumBuilder = ImmutableList.builder();
    List<IdOnlyContainerResource> albumIds = new ArrayList<>();
    int page = paginationData == null ? 0 : ((IntPaginationToken) paginationData).getStart();
    String url = format(ALBUMS_URL_TEMPLATE, page);
    List<Map<String, Object>> items = requestData(authData, url);
    // Request result doesn't indicate if it's the last page
    boolean hasMore = (items != null && items.size() != 0);
    for (Map<String, Object> item : items) {
        albumBuilder.add(new PhotoAlbum((String) item.get("id"), (String) item.get("title"), (String) item.get("description")));
        // Save album id for recalling export to get all the photos in albums
        albumIds.add(new IdOnlyContainerResource((String) item.get("id")));
    }
    if (page == 0) {
        // For checking non-album photos. Their export should be performed after all the others
        // Album will be created later
        albumIds.add(new IdOnlyContainerResource(DEFAULT_ALBUM_ID));
    }
    PaginationData newPage = null;
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
        int start = ((IntPaginationToken) newPage).getStart();
        monitor.info(() -> format("albums size: %s, newPage: %s", items.size(), start));
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albumBuilder.build(), null);
    ContinuationData continuationData = new ContinuationData(newPage);
    albumIds.forEach(continuationData::addContainerResource);
    ExportResult.ResultType resultType = ExportResult.ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ExportResult.ResultType.END;
    }
    return new ExportResult<>(resultType, photosContainerResource, continuationData);
}

Example 3 with IntPaginationToken

use of org.datatransferproject.types.common.IntPaginationToken in project data-transfer-project by google.

the class ImgurPhotoExporterTest method testPagination.

@Test
public void testPagination() throws Exception {
    server.enqueue(new MockResponse().setBody(page1Response));
    server.enqueue(new MockResponse().setBody(page2Response));
    int page = 0;
    ExportResult<PhotosContainerResource> page1Result = exporter.export(UUID.randomUUID(), token, Optional.of(new ExportInformation(new IntPaginationToken(page), new IdOnlyContainerResource(ImgurPhotosExporter.DEFAULT_ALBUM_ID))));
    page++;
    PhotosContainerResource page1Resource = page1Result.getExportedData();
    // 1th request returns 10 photos
    assertEquals(10, page1Resource.getPhotos().size());
    assertEquals(page, ((IntPaginationToken) page1Result.getContinuationData().getPaginationData()).getStart());
    ExportResult<PhotosContainerResource> page2Result = exporter.export(UUID.randomUUID(), token, Optional.of(new ExportInformation(new IntPaginationToken(page), new IdOnlyContainerResource(ImgurPhotosExporter.DEFAULT_ALBUM_ID))));
    page++;
    PhotosContainerResource page2Resource = page2Result.getExportedData();
    // 2th request returns 2 photos
    assertEquals(2, page2Resource.getPhotos().size());
    assertEquals(page, ((IntPaginationToken) page2Result.getContinuationData().getPaginationData()).getStart());
}

Example 4 with IntPaginationToken

use of org.datatransferproject.types.common.IntPaginationToken in project data-transfer-project by google.

the class ContinuationDataTest method verifySerializeDeserialize.

@Test
public void verifySerializeDeserialize() throws Exception {
    ObjectMapper objectMapper = new ObjectMapper();
    objectMapper.registerSubtypes(ContinuationData.class, IntPaginationToken.class, IdOnlyContainerResource.class);
    ContinuationData continuationData = new ContinuationData(new IntPaginationToken(100));
    continuationData.addContainerResource(new IdOnlyContainerResource("123"));
    String serialized = objectMapper.writeValueAsString(continuationData);
    ContinuationData deserialized = objectMapper.readValue(serialized, ContinuationData.class);
    Assert.assertNotNull(deserialized);
    Assert.assertEquals(100, ((IntPaginationToken) deserialized.getPaginationData()).getStart());
    Assert.assertEquals("123", ((IdOnlyContainerResource) deserialized.getContainerResources().get(0)).getId());
}

Example 5 with IntPaginationToken

use of org.datatransferproject.types.common.IntPaginationToken in project data-transfer-project by google.

the class ImgurPhotosExporter method requestNonAlbumPhotos.

/**
 * Queries all photos for the account. Chooses photos which are not included to the collection of
 * photos from albums.
 *
 * @param authData authentication information
 * @param paginationData pagination information to use for subsequent calls.
 */
private ExportResult<PhotosContainerResource> requestNonAlbumPhotos(TokensAndUrlAuthData authData, PaginationData paginationData, UUID jobId) throws IOException {
    int page = paginationData == null ? 0 : ((IntPaginationToken) paginationData).getStart();
    String url = format(ALL_PHOTOS_URL_TEMPLATE, page);
    Set<PhotoAlbum> albums = new HashSet<>();
    List<PhotoModel> photos = new ArrayList<>();
    List<Map<String, Object>> items = requestData(authData, url);
    boolean hasMore = (items != null && items.size() != 0);
    for (Map<String, Object> item : items) {
        String photoId = (String) item.get("id");
        // Select photos which are not included to the collection of retrieved album photos
        if (!albumPhotos.contains(photoId)) {
            PhotoModel photoModel = new PhotoModel((String) item.get("name"), (String) item.get("link"), (String) item.get("description"), (String) item.get("type"), (String) item.get("id"), DEFAULT_ALBUM_ID, true);
            photos.add(photoModel);
            InputStream inputStream = getImageAsStream(photoModel.getFetchableUrl());
            jobStore.create(jobId, photoModel.getFetchableUrl(), inputStream);
        }
    }
    if (!containsNonAlbumPhotos && photos.size() > 0) {
        // Add album for non-album photos
        albums.add(new PhotoAlbum(DEFAULT_ALBUM_ID, "Non-album photos", "Contains non-album photos"));
        // Make sure album will not be added multiply times on subsequent calls
        containsNonAlbumPhotos = true;
    }
    PaginationData newPage = null;
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
        monitor.info(() -> format("added non-album photos, size: %s", photos.size()));
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albums, photos);
    ContinuationData continuationData = new ContinuationData(newPage);
    ExportResult.ResultType resultType = ExportResult.ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ExportResult.ResultType.END;
    }
    return new ExportResult<>(resultType, photosContainerResource, continuationData);
}