Examples with ContinuationData org.datatransferproject.spi.transfer.types.ContinuationData used on opensource projects

Example 41 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class TwitterPhotosExporter method export.

@Override
public ExportResult<PhotosContainerResource> export(UUID jobId, TokenSecretAuthData authData, Optional<ExportInformation> exportInformation) {
    Twitter twitterApi = TwitterApiWrapper.getInstance(appCredentials, authData);
    int pageNumber = 1;
    if (exportInformation.isPresent()) {
        IntPaginationToken pageToken = (IntPaginationToken) exportInformation.get().getPaginationData();
        if (pageToken != null && pageToken.getStart() > 1) {
            pageNumber = pageToken.getStart();
        }
    }
    Paging paging = new Paging(pageNumber, PAGE_SIZE);
    try {
        String page = "" + pageNumber;
        long id = twitterApi.getId();
        monitor.debug(() -> format("Getting tweets for %s (page %s)", id, page));
        ResponseList<Status> statuses = twitterApi.getUserTimeline(id, paging);
        List<PhotoModel> photos = new ArrayList<>();
        for (Status status : statuses) {
            boolean hasMedia = status.getMediaEntities().length > 0;
            if (hasMedia && !status.isRetweet()) {
                for (MediaEntity mediaEntity : status.getMediaEntities()) {
                    photos.add(new PhotoModel("Twitter Photo " + mediaEntity.getId(), mediaEntity.getMediaURL(), status.getText(), null, Long.toString(status.getId()), null, false));
                }
            }
        }
        boolean moreData = statuses.size() == PAGE_SIZE;
        return new ExportResult<>(moreData ? ResultType.CONTINUE : ResultType.END, new PhotosContainerResource(null, photos), moreData ? new ContinuationData(new IntPaginationToken(pageNumber + 1)) : null);
    } catch (TwitterException e) {
        return new ExportResult<>(e);
    }
}

Example 42 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class SmugMugPhotosExporter method exportAlbums.

private ExportResult<PhotosContainerResource> exportAlbums(StringPaginationToken paginationData, SmugMugInterface smugMugInterface) throws IOException {
    SmugMugAlbumsResponse albumsResponse;
    try {
        // Make request to SmugMug
        String albumInfoUri = "";
        if (paginationData != null) {
            String pageToken = paginationData.getToken();
            Preconditions.checkState(pageToken.startsWith(ALBUM_TOKEN_PREFIX), "Invalid pagination token " + pageToken);
            albumInfoUri = pageToken.substring(ALBUM_TOKEN_PREFIX.length());
        }
        albumsResponse = smugMugInterface.getAlbums(albumInfoUri);
    } catch (IOException e) {
        monitor.severe(() -> "Unable to get AlbumsResponse: ", e);
        throw e;
    }
    // Set up continuation data
    StringPaginationToken paginationToken = null;
    if (albumsResponse.getPageInfo() != null && albumsResponse.getPageInfo().getNextPage() != null) {
        paginationToken = new StringPaginationToken(ALBUM_TOKEN_PREFIX + albumsResponse.getPageInfo().getNextPage());
    }
    ContinuationData continuationData = new ContinuationData(paginationToken);
    // Build album list
    List<PhotoAlbum> albumsList = new ArrayList<>();
    if (albumsResponse.getAlbums() != null) {
        for (SmugMugAlbum album : albumsResponse.getAlbums()) {
            albumsList.add(new PhotoAlbum(album.getUri(), album.getName(), album.getDescription()));
            continuationData.addContainerResource(new IdOnlyContainerResource(album.getUri()));
        }
    }
    PhotosContainerResource resource = new PhotosContainerResource(albumsList, null);
    // Get result type
    ResultType resultType = ResultType.CONTINUE;
    if (paginationToken == null) {
        resultType = ResultType.END;
    }
    return new ExportResult<>(resultType, resource, continuationData);
}

Example 43 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class SmugMugPhotosExporter method exportPhotos.

private ExportResult<PhotosContainerResource> exportPhotos(IdOnlyContainerResource containerResource, StringPaginationToken paginationData, SmugMugInterface smugMugInterface, UUID jobId) throws IOException {
    List<PhotoModel> photoList = new ArrayList<>();
    // Make request to SmugMug
    String photoInfoUri;
    if (paginationData != null) {
        String token = paginationData.getToken();
        Preconditions.checkState(token.startsWith(PHOTO_TOKEN_PREFIX), "Invalid pagination token " + token);
        photoInfoUri = token.substring(PHOTO_TOKEN_PREFIX.length());
    } else {
        photoInfoUri = containerResource.getId();
    }
    SmugMugAlbumImageResponse albumImageList;
    try {
        albumImageList = smugMugInterface.getListOfAlbumImages(photoInfoUri + "!images");
    } catch (IOException e) {
        monitor.severe(() -> "Unable to get SmugMugAlbumImageResponse");
        throw e;
    }
    // Set up continuation data
    StringPaginationToken pageToken = null;
    if (albumImageList.getPageInfo().getNextPage() != null) {
        pageToken = new StringPaginationToken(PHOTO_TOKEN_PREFIX + albumImageList.getPageInfo().getNextPage());
    }
    ContinuationData continuationData = new ContinuationData(pageToken);
    // Make list of photos - images may be empty if the album provided is empty
    List<SmugMugImage> images = albumImageList.getAlbumImages() == null ? ImmutableList.of() : albumImageList.getAlbumImages();
    for (SmugMugImage albumImage : images) {
        if (!albumImage.isPhoto()) {
            continue;
        }
        String title = albumImage.getTitle();
        if (Strings.isNullOrEmpty(title)) {
            title = albumImage.getFileName();
        }
        PhotoModel model = new PhotoModel(title, albumImage.getArchivedUri(), albumImage.getCaption(), getMimeType(albumImage.getFormat()), albumImage.getArchivedUri(), containerResource.getId(), true);
        InputStream inputStream = smugMugInterface.getImageAsStream(model.getFetchableUrl());
        jobStore.create(jobId, model.getFetchableUrl(), inputStream);
        photoList.add(model);
    }
    PhotosContainerResource resource = new PhotosContainerResource(null, photoList);
    // Get result type
    ResultType resultType = ResultType.CONTINUE;
    if (pageToken == null) {
        resultType = ResultType.END;
    }
    return new ExportResult<>(resultType, resource, continuationData);
}

Example 44 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class FlickrPhotosExporterTest method exportAlbumInitial.

@Test
public void exportAlbumInitial() throws FlickrException {
    // set up auth, flickr service
    when(user.getId()).thenReturn("userId");
    when(authInterface.checkToken(any(Token.class))).thenReturn(auth);
    when(flickr.getPhotosetsInterface()).thenReturn(photosetsInterface);
    when(flickr.getPhotosInterface()).thenReturn(photosInterface);
    when(flickr.getAuthInterface()).thenReturn(authInterface);
    // setup photoset
    Photoset photoset = FlickrTestUtils.initializePhotoset("photosetId", "title", "description");
    // setup photoset list (aka album view)
    int page = 1;
    Photosets photosetsList = new Photosets();
    photosetsList.setPage(page);
    photosetsList.setPages(page + 1);
    photosetsList.setPhotosets(Collections.singletonList(photoset));
    when(photosetsInterface.getList(anyString(), anyInt(), anyInt(), anyString())).thenReturn(photosetsList);
    // run test
    FlickrPhotosExporter exporter = new FlickrPhotosExporter(flickr, TransferServiceConfig.getDefaultInstance());
    AuthData authData = new TokenSecretAuthData("token", "secret");
    ExportResult<PhotosContainerResource> result = exporter.export(UUID.randomUUID(), authData, Optional.empty());
    // make sure album and photo information is correct
    assertThat(result.getExportedData().getPhotos()).isEmpty();
    Collection<PhotoAlbum> albums = result.getExportedData().getAlbums();
    assertThat(albums.size()).isEqualTo(1);
    assertThat(albums).containsExactly(new PhotoAlbum("photosetId", "title", "description"));
    // check continuation information
    ContinuationData continuationData = (ContinuationData) result.getContinuationData();
    assertThat(continuationData.getPaginationData()).isInstanceOf(IntPaginationToken.class);
    assertThat(((IntPaginationToken) continuationData.getPaginationData()).getStart()).isEqualTo(page + 1);
    Collection<? extends ContainerResource> subResources = continuationData.getContainerResources();
    assertThat(subResources.size()).isEqualTo(1);
    assertThat(subResources).containsExactly(new IdOnlyContainerResource("photosetId"));
}

Example 45 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class FlickrPhotosExporterTest method exportPhotosFromPhotoset.

@Test
public void exportPhotosFromPhotoset() throws FlickrException {
    // set up auth, flickr service
    when(user.getId()).thenReturn("userId");
    when(authInterface.checkToken(any(Token.class))).thenReturn(auth);
    when(flickr.getPhotosetsInterface()).thenReturn(photosetsInterface);
    when(flickr.getPhotosInterface()).thenReturn(photosInterface);
    when(flickr.getAuthInterface()).thenReturn(authInterface);
    // getting photos from a set with id photosetsId and page 1
    int page = 1;
    String photosetsId = "photosetsId";
    ExportInformation exportInformation = new ExportInformation(null, new IdOnlyContainerResource(photosetsId));
    // make lots of photos and add them to PhotoList (also adding pagination information)
    int numPhotos = 4;
    PhotoList<Photo> photosList = new PhotoList<>();
    for (int i = 0; i < numPhotos; i++) {
        photosList.add(FlickrTestUtils.initializePhoto("title" + 1, "url" + i, "description" + i, MEDIA_TYPE));
    }
    photosList.setPage(page);
    photosList.setPages(page + 1);
    when(photosetsInterface.getPhotos(anyString(), anySet(), anyInt(), anyInt(), anyInt())).thenReturn(photosList);
    // run test
    FlickrPhotosExporter exporter = new FlickrPhotosExporter(flickr, TransferServiceConfig.getDefaultInstance());
    ExportResult<PhotosContainerResource> result = exporter.export(UUID.randomUUID(), new TokenSecretAuthData("token", "secret"), Optional.of(exportInformation));
    assertThat(result.getExportedData().getPhotos().size()).isEqualTo(numPhotos);
    assertThat(result.getExportedData().getAlbums()).isEmpty();
    ContinuationData continuationData = (ContinuationData) result.getContinuationData();
    assertThat(continuationData.getContainerResources()).isEmpty();
    assertThat(((IntPaginationToken) continuationData.getPaginationData()).getStart()).isEqualTo(page + 1);
}