Examples with PaginationData org.datatransferproject.types.common.PaginationData used on opensource projects

Example 1 with PaginationData

use of org.datatransferproject.types.common.PaginationData in project data-transfer-project by google.

the class FacebookPhotosExporter method exportAlbums.

private ExportResult<PhotosContainerResource> exportAlbums(TokensAndUrlAuthData authData, Optional<StringPaginationToken> paginationData) throws CopyExceptionWithFailureReason {
    Optional<String> paginationToken = stripTokenPrefix(paginationData, ALBUM_TOKEN_PREFIX);
    // Get albums
    Connection<Album> connection = getOrCreatePhotosInterface(authData).getAlbums(paginationToken);
    PaginationData nextPageData = null;
    String token = connection.getAfterCursor();
    if (!Strings.isNullOrEmpty(token)) {
        nextPageData = new StringPaginationToken(ALBUM_TOKEN_PREFIX + token);
    }
    ContinuationData continuationData = new ContinuationData(nextPageData);
    List<Album> albums = connection.getData();
    if (albums.isEmpty()) {
        return new ExportResult<>(ExportResult.ResultType.END, null, null);
    }
    ArrayList<PhotoAlbum> exportAlbums = new ArrayList<>();
    for (Album album : albums) {
        exportAlbums.add(new PhotoAlbum(album.getId(), album.getName(), album.getDescription()));
        continuationData.addContainerResource(new IdOnlyContainerResource(album.getId()));
    }
    return new ExportResult<>(ExportResult.ResultType.CONTINUE, new PhotosContainerResource(exportAlbums, null), continuationData);
}

Example 2 with PaginationData

use of org.datatransferproject.types.common.PaginationData in project data-transfer-project by google.

the class FacebookPhotosExporter method exportPhotos.

private ExportResult<PhotosContainerResource> exportPhotos(UUID jobId, TokensAndUrlAuthData authData, IdOnlyContainerResource containerResource, Optional<StringPaginationToken> paginationData) throws CopyExceptionWithFailureReason {
    Optional<String> paginationToken = stripTokenPrefix(paginationData, PHOTO_TOKEN_PREFIX);
    String albumId = containerResource.getId();
    try {
        Connection<Photo> photoConnection = getOrCreatePhotosInterface(authData).getPhotos(albumId, paginationToken);
        List<Photo> photos = photoConnection.getData();
        if (photos.isEmpty()) {
            return new ExportResult<>(ExportResult.ResultType.END, null);
        }
        ArrayList<PhotoModel> exportPhotos = new ArrayList<>();
        for (Photo photo : photos) {
            final String url = photo.getImages().get(0).getSource();
            final String fbid = photo.getId();
            if (null == url || url.isEmpty()) {
                monitor.severe(() -> String.format("Source was missing or empty for photo %s", fbid));
                continue;
            }
            boolean photoWasGarbage;
            try {
                photoWasGarbage = modifyExifAndStorePhoto(jobId, photo, url, photo.getId());
            } catch (IOException e) {
                monitor.info(() -> String.format("Error while modifying exif or storing photo %s", fbid), e);
                photoWasGarbage = true;
            }
            if (photoWasGarbage) {
                continue;
            }
            exportPhotos.add(new PhotoModel(String.format("%s.jpg", photo.getId()), // store and the url is too long for that.
            photo.getId(), photo.getName(), "image/jpg", photo.getId(), albumId, true, photo.getCreatedTime()));
        }
        String token = photoConnection.getAfterCursor();
        if (Strings.isNullOrEmpty(token)) {
            return new ExportResult<>(ExportResult.ResultType.END, new PhotosContainerResource(null, exportPhotos));
        } else {
            PaginationData nextPageData = new StringPaginationToken(PHOTO_TOKEN_PREFIX + token);
            ContinuationData continuationData = new ContinuationData(nextPageData);
            return new ExportResult<>(ExportResult.ResultType.CONTINUE, new PhotosContainerResource(null, exportPhotos), continuationData);
        }
    } catch (FacebookGraphException e) {
        String message = e.getMessage();
        // In such case, we should skip this object and continue with the rest of the transfer.
        if (message != null && message.contains("code 100, subcode 33")) {
            monitor.info(() -> "Cannot find photos to export, skipping to the next bunch", e);
            return new ExportResult<>(ExportResult.ResultType.END, null);
        }
        throw e;
    }
}

Example 3 with PaginationData

use of org.datatransferproject.types.common.PaginationData in project data-transfer-project by google.

the class FacebookVideosExporter method exportVideos.

private ExportResult<VideosContainerResource> exportVideos(TokensAndUrlAuthData authData, Optional<StringPaginationToken> paginationData) throws CopyExceptionWithFailureReason {
    Optional<String> paginationToken = paginationData.map(StringPaginationToken::getToken);
    try {
        Connection<Video> videoConnection = getOrCreateVideosInterface(authData).getVideos(paginationToken);
        List<Video> videos = videoConnection.getData();
        if (videos.isEmpty()) {
            return new ExportResult<>(ExportResult.ResultType.END, null);
        }
        ArrayList<VideoModel> exportVideos = new ArrayList<>();
        for (Video video : videos) {
            final String url = video.getSource();
            final String fbid = video.getId();
            if (null == url || url.isEmpty()) {
                monitor.severe(() -> String.format("Source was missing or empty for video %s", fbid));
                continue;
            }
            exportVideos.add(new VideoModel(String.format("%s.mp4", fbid), url, video.getDescription(), "video/mp4", fbid, null, false));
        }
        String token = videoConnection.getAfterCursor();
        if (Strings.isNullOrEmpty(token)) {
            return new ExportResult<>(ExportResult.ResultType.END, new VideosContainerResource(null, exportVideos));
        } else {
            PaginationData nextPageData = new StringPaginationToken(token);
            ContinuationData continuationData = new ContinuationData(nextPageData);
            return new ExportResult<>(ExportResult.ResultType.CONTINUE, new VideosContainerResource(null, exportVideos), continuationData);
        }
    } catch (FacebookGraphException e) {
        String message = e.getMessage();
        // In such case, we should skip this object and continue with the rest of the transfer.
        if (message != null && message.contains("code 100, subcode 33")) {
            monitor.info(() -> "Cannot find videos to export, skipping to the next bunch", e);
            return new ExportResult<>(ExportResult.ResultType.END, null);
        }
        throw e;
    }
}

Example 4 with PaginationData

use of org.datatransferproject.types.common.PaginationData in project data-transfer-project by google.

the class MicrosoftPhotosExporter method exportOneDrivePhotos.

@VisibleForTesting
ExportResult<PhotosContainerResource> exportOneDrivePhotos(TokensAndUrlAuthData authData, Optional<IdOnlyContainerResource> albumData, Optional<PaginationData> paginationData, UUID jobId) throws IOException {
    Optional<String> albumId = Optional.empty();
    if (albumData.isPresent()) {
        albumId = Optional.of(albumData.get().getId());
    }
    Optional<String> paginationUrl = getDrivePaginationToken(paginationData);
    MicrosoftDriveItemsResponse driveItemsResponse;
    if (paginationData.isPresent() || albumData.isPresent()) {
        driveItemsResponse = getOrCreatePhotosInterface(authData).getDriveItems(albumId, paginationUrl);
    } else {
        driveItemsResponse = getOrCreatePhotosInterface(authData).getDriveItemsFromSpecialFolder(MicrosoftSpecialFolder.FolderType.photos);
    }
    PaginationData nextPageData = SetNextPageToken(driveItemsResponse);
    ContinuationData continuationData = new ContinuationData(nextPageData);
    PhotosContainerResource containerResource;
    MicrosoftDriveItem[] driveItems = driveItemsResponse.getDriveItems();
    List<PhotoAlbum> albums = new ArrayList<>();
    List<PhotoModel> photos = new ArrayList<>();
    if (driveItems != null && driveItems.length > 0) {
        for (MicrosoftDriveItem driveItem : driveItems) {
            PhotoAlbum album = tryConvertDriveItemToPhotoAlbum(driveItem, jobId);
            if (album != null) {
                albums.add(album);
                continuationData.addContainerResource(new IdOnlyContainerResource(driveItem.id));
            }
            PhotoModel photo = tryConvertDriveItemToPhotoModel(albumId, driveItem, jobId);
            if (photo != null) {
                photos.add(photo);
            }
        }
    }
    ExportResult.ResultType result = nextPageData == null ? ExportResult.ResultType.END : ExportResult.ResultType.CONTINUE;
    containerResource = new PhotosContainerResource(albums, photos);
    return new ExportResult<>(result, containerResource, continuationData);
}

Example 5 with PaginationData

use of org.datatransferproject.types.common.PaginationData in project data-transfer-project by google.

the class PortabilityInMemoryDataCopierTest method continuationDataWithPaginationDataAndMultipleSubResources.

@Test
public void continuationDataWithPaginationDataAndMultipleSubResources() throws CopyException, IOException {
    // Arrange
    PaginationData paginationData = Mockito.mock(PaginationData.class);
    ContainerResource subResource1 = Mockito.mock(ContainerResource.class);
    ContainerResource subResource2 = Mockito.mock(ContainerResource.class);
    ExportInformation paginationExportInfo = new ExportInformation(paginationData, null);
    ExportInformation subResource1ExportInfo = new ExportInformation(null, subResource1);
    ExportInformation subResource2ExportInfo = new ExportInformation(null, subResource2);
    Mockito.when(continuationData.getPaginationData()).thenReturn(paginationData);
    Mockito.when(continuationData.getContainerResources()).thenReturn(Arrays.asList(subResource1, subResource2));
    Mockito.when(initialExportResult.getContinuationData()).thenReturn(continuationData);
    Mockito.doReturn(initialExportResult).when(inMemoryDataCopier).copyIteration(jobId, exportAuthData, importAuthData, Optional.of(exportInfo), jobIdPrefix, 1);
    // Act
    inMemoryDataCopier.copy(exportAuthData, importAuthData, jobId, Optional.of(exportInfo));
    // Assert
    InOrder orderVerifier = Mockito.inOrder(inMemoryDataCopier);
    orderVerifier.verify(inMemoryDataCopier).copyIteration(jobId, exportAuthData, importAuthData, Optional.of(exportInfo), jobIdPrefix, 1);
    orderVerifier.verify(inMemoryDataCopier).copyIteration(jobId, exportAuthData, importAuthData, Optional.of(paginationExportInfo), jobIdPrefix, 2);
    orderVerifier.verify(inMemoryDataCopier).copyIteration(jobId, exportAuthData, importAuthData, Optional.of(subResource1ExportInfo), jobIdPrefix, 3);
    orderVerifier.verify(inMemoryDataCopier).copyIteration(jobId, exportAuthData, importAuthData, Optional.of(subResource2ExportInfo), jobIdPrefix, 4);
}