Examples with ExportResult org.datatransferproject.spi.transfer.provider.ExportResult used on opensource projects

Example 41 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class FlickrPhotosExporter method getAlbums.

private ExportResult<PhotosContainerResource> getAlbums(PaginationData paginationData, Auth auth) {
    ImmutableList.Builder<PhotoAlbum> albumBuilder = ImmutableList.builder();
    List<IdOnlyContainerResource> subResources = new ArrayList<>();
    int page = paginationData == null ? 1 : ((IntPaginationToken) paginationData).getStart();
    Photosets photoSetList;
    try {
        perUserRateLimiter.acquire();
        photoSetList = photosetsInterface.getList(auth.getUser().getId(), PHOTO_SETS_PER_PAGE, page, PHOTOSET_EXTRAS);
    } catch (FlickrException e) {
        return new ExportResult<>(e);
    }
    for (Photoset photoSet : photoSetList.getPhotosets()) {
        // Saving data to the album allows the target service to recreate the album structure
        albumBuilder.add(new PhotoAlbum(photoSet.getId(), photoSet.getTitle(), photoSet.getDescription()));
        // Adding subresources tells the framework to recall export to get all the photos
        subResources.add(new IdOnlyContainerResource(photoSet.getId()));
    }
    PaginationData newPage = null;
    boolean hasMore = photoSetList.getPage() != photoSetList.getPages() && !photoSetList.getPhotosets().isEmpty();
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
    } else {
        // No more albums to get, add a resource for albumless items
        subResources.add(new IdOnlyContainerResource(""));
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albumBuilder.build(), null);
    ContinuationData continuationData = new ContinuationData(newPage);
    subResources.forEach(resource -> continuationData.addContainerResource(resource));
    // Get result type
    ResultType resultType = ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ResultType.END;
    }
    return new ExportResult<>(resultType, photosContainerResource, continuationData);
}

Example 42 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class DriveExporter method export.

@Override
public ExportResult<BlobbyStorageContainerResource> export(UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> optionalExportInformation) throws Exception {
    Drive driveInterface = getDriveInterface((authData));
    List driveListOperation = driveInterface.files().list();
    // If the folder Id isn't specified then use root
    String parentId = "root";
    if (optionalExportInformation.isPresent()) {
        ExportInformation exportInformation = optionalExportInformation.get();
        if (exportInformation.getPaginationData() != null) {
            StringPaginationToken paginationToken = (StringPaginationToken) exportInformation.getPaginationData();
            driveListOperation.setPageToken(paginationToken.getToken());
        }
        if (exportInformation.getContainerResource() != null) {
            BlobbyStorageContainerResource parent = (BlobbyStorageContainerResource) exportInformation.getContainerResource();
            parentId = parent.getId();
        }
    }
    driveListOperation.setFields("files(id, name, modifiedTime, mimeType)").setQ(String.format(DRIVE_QUERY_FORMAT, parentId));
    ArrayList<DigitalDocumentWrapper> files = new ArrayList<>();
    ArrayList<BlobbyStorageContainerResource> folders = new ArrayList<>();
    FileList fileList = driveListOperation.execute();
    for (File file : fileList.getFiles()) {
        if (FOLDER_MIME_TYPE.equals(file.getMimeType())) {
            folders.add(new BlobbyStorageContainerResource(file.getName(), file.getId(), null, null));
        } else if (FUSION_TABLE_MIME_TYPE.equals(file.getMimeType())) {
            monitor.info(() -> "Exporting of fusion tables is not yet supported: " + file);
        } else if (MAP_MIME_TYPE.equals(file.getMimeType())) {
            monitor.info(() -> "Exporting of maps is not yet supported: " + file);
        } else {
            try {
                InputStream inputStream;
                String newMimeType = file.getMimeType();
                if (EXPORT_FORMATS.containsKey(file.getMimeType())) {
                    newMimeType = EXPORT_FORMATS.get(file.getMimeType());
                    inputStream = driveInterface.files().export(file.getId(), newMimeType).executeMedia().getContent();
                } else {
                    inputStream = driveInterface.files().get(file.getId()).setAlt("media").executeMedia().getContent();
                }
                jobStore.create(jobId, file.getId(), inputStream);
                files.add(new DigitalDocumentWrapper(new DtpDigitalDocument(file.getName(), file.getModifiedTime().toStringRfc3339(), newMimeType), file.getMimeType(), file.getId()));
            } catch (Exception e) {
                monitor.severe(() -> "Error exporting " + file, e);
            }
        }
        monitor.info(() -> "Exported " + file);
    }
    ResultType resultType = isDone(fileList) ? ResultType.END : ResultType.CONTINUE;
    BlobbyStorageContainerResource result = new BlobbyStorageContainerResource(null, parentId, files, folders);
    StringPaginationToken paginationToken = null;
    if (!Strings.isNullOrEmpty(fileList.getNextPageToken())) {
        paginationToken = new StringPaginationToken(fileList.getNextPageToken());
    }
    ContinuationData continuationData = new ContinuationData(paginationToken);
    folders.forEach(continuationData::addContainerResource);
    return new ExportResult<>(resultType, result, continuationData);
}

Example 43 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class GooglePhotosExporter method exportPhotos.

@VisibleForTesting
ExportResult<PhotosContainerResource> exportPhotos(TokensAndUrlAuthData authData, Optional<IdOnlyContainerResource> albumData, Optional<PaginationData> paginationData, UUID jobId) throws IOException, InvalidTokenException, PermissionDeniedException {
    Optional<String> albumId = Optional.empty();
    if (albumData.isPresent()) {
        albumId = Optional.of(albumData.get().getId());
    }
    Optional<String> paginationToken = getPhotosPaginationToken(paginationData);
    MediaItemSearchResponse mediaItemSearchResponse = getOrCreatePhotosInterface(authData).listMediaItems(albumId, paginationToken);
    PaginationData nextPageData = null;
    if (!Strings.isNullOrEmpty(mediaItemSearchResponse.getNextPageToken())) {
        nextPageData = new StringPaginationToken(PHOTO_TOKEN_PREFIX + mediaItemSearchResponse.getNextPageToken());
    }
    ContinuationData continuationData = new ContinuationData(nextPageData);
    PhotosContainerResource containerResource = null;
    GoogleMediaItem[] mediaItems = mediaItemSearchResponse.getMediaItems();
    if (mediaItems != null && mediaItems.length > 0) {
        List<PhotoModel> photos = convertPhotosList(albumId, mediaItems, jobId);
        containerResource = new PhotosContainerResource(null, photos);
    }
    ResultType resultType = ResultType.CONTINUE;
    if (nextPageData == null) {
        resultType = ResultType.END;
    }
    return new ExportResult<>(resultType, containerResource, continuationData);
}

Example 44 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class GooglePhotosExporter method exportPhotosContainer.

private ExportResult<PhotosContainerResource> exportPhotosContainer(PhotosContainerResource container, TokensAndUrlAuthData authData) throws IOException, InvalidTokenException, PermissionDeniedException {
    ImmutableList.Builder<PhotoAlbum> albumBuilder = ImmutableList.builder();
    ImmutableList.Builder<PhotoModel> photosBuilder = ImmutableList.builder();
    List<IdOnlyContainerResource> subResources = new ArrayList<>();
    for (PhotoAlbum album : container.getAlbums()) {
        GoogleAlbum googleAlbum = getOrCreatePhotosInterface(authData).getAlbum(album.getId());
        albumBuilder.add(new PhotoAlbum(googleAlbum.getId(), googleAlbum.getTitle(), null));
        // Adding subresources tells the framework to recall export to get all the photos
        subResources.add(new IdOnlyContainerResource(googleAlbum.getId()));
    }
    for (PhotoModel photo : container.getPhotos()) {
        GoogleMediaItem googleMediaItem = getOrCreatePhotosInterface(authData).getMediaItem(photo.getDataId());
        photosBuilder.add(convertToPhotoModel(Optional.empty(), googleMediaItem));
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albumBuilder.build(), photosBuilder.build());
    ContinuationData continuationData = new ContinuationData(null);
    subResources.forEach(resource -> continuationData.addContainerResource(resource));
    return new ExportResult<>(ResultType.CONTINUE, photosContainerResource, continuationData);
}