Search in sources :

Example 1 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class FacebookPhotosExporter method export.

@Override
public ExportResult<PhotosContainerResource> export(UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation) throws CopyExceptionWithFailureReason {
    Preconditions.checkNotNull(authData);
    if (!exportInformation.isPresent()) {
        // Start by getting the list of albums to export
        return exportAlbums(authData, Optional.empty());
    }
    StringPaginationToken paginationToken = (StringPaginationToken) exportInformation.get().getPaginationData();
    ContainerResource containerResource = exportInformation.get().getContainerResource();
    boolean containerResourcePresent = containerResource != null;
    boolean paginationDataPresent = paginationToken != null;
    if (!containerResourcePresent && paginationDataPresent && paginationToken.getToken().startsWith(ALBUM_TOKEN_PREFIX)) {
        // Continue exporting albums
        return exportAlbums(authData, Optional.of(paginationToken));
    } else if (containerResourcePresent && containerResource instanceof PhotosContainerResource) {
        // We have had albums specified from the front end so process them for import
        PhotosContainerResource photosContainerResource = (PhotosContainerResource) containerResource;
        Preconditions.checkNotNull(photosContainerResource.getAlbums());
        ContinuationData continuationData = new ContinuationData(null);
        for (PhotoAlbum album : photosContainerResource.getAlbums()) {
            continuationData.addContainerResource(new IdOnlyContainerResource(album.getId()));
        }
        return new ExportResult<>(ExportResult.ResultType.CONTINUE, photosContainerResource, continuationData);
    } else if (containerResourcePresent && containerResource instanceof IdOnlyContainerResource) {
        // Export photos
        return exportPhotos(jobId, authData, (IdOnlyContainerResource) containerResource, Optional.ofNullable(paginationToken));
    } else {
        throw new IllegalStateException(String.format("Invalid state passed into FacebookPhotosExporter. ExportInformation: %s", exportInformation));
    }
}
Also used : PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) ContainerResource(org.datatransferproject.types.common.models.ContainerResource) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) ContinuationData(org.datatransferproject.spi.transfer.types.ContinuationData) PhotoAlbum(org.datatransferproject.types.common.models.photos.PhotoAlbum) StringPaginationToken(org.datatransferproject.types.common.StringPaginationToken)

Example 2 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class PortabilityStackInMemoryDataCopier method updateStackAfterCopyIteration.

private void updateStackAfterCopyIteration(UUID jobId, String jobIdPrefix, ContainerResource exportContainerResource, int copyIteration, ContinuationData continuationData) {
    if (null != continuationData) {
        // Start processing sub-resources
        if (continuationData.getContainerResources() != null && !continuationData.getContainerResources().isEmpty()) {
            List<ContainerResource> subResources = continuationData.getContainerResources();
            for (int i = subResources.size() - 1; i >= 0; i--) {
                monitor.debug(() -> jobIdPrefix + "Pushing to the stack a new copy iteration with a new container resource, copy iteration: " + copyIteration);
                exportInfoStack.push((new ExportInformation(null, subResources.get(i))));
            }
        }
        // Push the next page of items onto the stack
        if (null != continuationData.getPaginationData()) {
            monitor.debug(() -> jobIdPrefix + "Pushing to the stack a new copy iteration with pagination info, copy iteration: " + copyIteration);
            exportInfoStack.push(new ExportInformation(continuationData.getPaginationData(), exportContainerResource));
        }
    }
    jobStore.storeJobStack(jobId, (Stack<ExportInformation>) exportInfoStack.clone());
}
Also used : ExportInformation(org.datatransferproject.types.common.ExportInformation) ContainerResource(org.datatransferproject.types.common.models.ContainerResource)

Example 3 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class PortabilityStackInMemoryDataCopier method copy.

/**
 * Transfers data from the given {@code exporter} optionally starting at the point specified in
 * the provided {@code exportInformation}. Imports the data using the provided {@code importer}.
 * If there is more data to required to be exported, recursively copies using the specific {@link
 * ExportInformation} to continue the process.
 *
 * @param exportAuthData The auth data for the export
 * @param importAuthData The auth data for the import
 * @param exportInfo Any pagination or resource information to use for subsequent calls.
 */
@Override
public Collection<ErrorDetail> copy(AuthData exportAuthData, AuthData importAuthData, UUID jobId, Optional<ExportInformation> exportInfo) throws IOException, CopyException {
    idempotentImportExecutor.setJobId(jobId);
    String jobIdPrefix = "Job " + jobId + ": ";
    Optional<Stack<ExportInformation>> maybeLoadedStack = jobStore.loadJobStack(jobId);
    if (maybeLoadedStack.isPresent()) {
        // load stack from partially completed transfer
        exportInfoStack = maybeLoadedStack.get();
    } else {
        // start new transfer
        int initialCopyIteration = COPY_ITERATION_COUNTER.incrementAndGet();
        ExportResult<?> initialExportResult = copyIteration(jobId, exportAuthData, importAuthData, exportInfo, jobIdPrefix, initialCopyIteration);
        // Import and Export were successful, determine what to do next
        ContainerResource exportContainerResource = exportInfo.isPresent() ? exportInfo.get().getContainerResource() : null;
        updateStackAfterCopyIteration(jobId, jobIdPrefix, exportContainerResource, initialCopyIteration, initialExportResult.getContinuationData());
    }
    while (!exportInfoStack.isEmpty()) {
        int copyIteration = COPY_ITERATION_COUNTER.incrementAndGet();
        ExportInformation currentExportInfo = exportInfoStack.pop();
        ExportResult<?> exportResult = copyIteration(jobId, exportAuthData, importAuthData, Optional.of(currentExportInfo), jobIdPrefix, copyIteration);
        // Import and Export were successful, determine what to do next
        updateStackAfterCopyIteration(jobId, jobIdPrefix, currentExportInfo.getContainerResource(), copyIteration, exportResult.getContinuationData());
    }
    return idempotentImportExecutor.getErrors();
}
Also used : ExportInformation(org.datatransferproject.types.common.ExportInformation) ContainerResource(org.datatransferproject.types.common.models.ContainerResource) Stack(java.util.Stack)

Example 4 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class MicrosoftPhotosExporterTest method exportOneAlbumWithNextPage.

@Test
public void exportOneAlbumWithNextPage() throws IOException {
    // Setup
    MicrosoftDriveItem folderItem = setUpSingleAlbum();
    when(driveItemsResponse.getDriveItems()).thenReturn(new MicrosoftDriveItem[] { folderItem });
    when(driveItemsResponse.getNextPageLink()).thenReturn(DRIVE_PAGE_URL);
    // Run
    ExportResult<PhotosContainerResource> result = microsoftPhotosExporter.exportOneDrivePhotos(null, Optional.empty(), Optional.empty(), uuid);
    // Verify method calls
    verify(photosInterface).getDriveItemsFromSpecialFolder(MicrosoftSpecialFolder.FolderType.photos);
    verify(driveItemsResponse).getDriveItems();
    // Verify pagination token is set
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(DRIVE_TOKEN_PREFIX + DRIVE_PAGE_URL);
    // Verify one album is ready for import
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums.stream().map(PhotoAlbum::getId).collect(Collectors.toList())).containsExactly(FOLDER_ID);
    // Verify photos should be empty (in the root)
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos).isEmpty();
    // Verify there is one container ready for sub-processing
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources.stream().map(a -> ((IdOnlyContainerResource) a).getId()).collect(Collectors.toList())).containsExactly(FOLDER_ID);
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ExportResult(org.datatransferproject.spi.transfer.provider.ExportResult) PhotoAlbum(org.datatransferproject.types.common.models.photos.PhotoAlbum) Matchers(org.mockito.Matchers) PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) StringPaginationToken(org.datatransferproject.types.common.StringPaginationToken) PhotoModel(org.datatransferproject.types.common.models.photos.PhotoModel) Before(org.junit.Before) JacksonFactory(com.google.api.client.json.jackson2.JacksonFactory) Collection(java.util.Collection) IOException(java.io.IOException) ContainerResource(org.datatransferproject.types.common.models.ContainerResource) Test(org.junit.Test) UUID(java.util.UUID) Truth.assertThat(com.google.common.truth.Truth.assertThat) Collectors(java.util.stream.Collectors) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) Mockito(org.mockito.Mockito) List(java.util.List) Monitor(org.datatransferproject.api.launcher.Monitor) TokensAndUrlAuthData(org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData) DRIVE_TOKEN_PREFIX(org.datatransferproject.transfer.microsoft.photos.MicrosoftPhotosExporter.DRIVE_TOKEN_PREFIX) Optional(java.util.Optional) ContinuationData(org.datatransferproject.spi.transfer.types.ContinuationData) MicrosoftCredentialFactory(org.datatransferproject.transfer.microsoft.common.MicrosoftCredentialFactory) org.datatransferproject.transfer.microsoft.driveModels(org.datatransferproject.transfer.microsoft.driveModels) PhotoModel(org.datatransferproject.types.common.models.photos.PhotoModel) ContinuationData(org.datatransferproject.spi.transfer.types.ContinuationData) PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) ContainerResource(org.datatransferproject.types.common.models.ContainerResource) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) PhotoAlbum(org.datatransferproject.types.common.models.photos.PhotoAlbum) StringPaginationToken(org.datatransferproject.types.common.StringPaginationToken) Test(org.junit.Test)

Example 5 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class MicrosoftPhotosExporterTest method exportPhotoWithNextPage.

@Test
public void exportPhotoWithNextPage() throws IOException {
    // Setup
    when(driveItemsResponse.getNextPageLink()).thenReturn(null);
    MicrosoftDriveItem photoItem = setUpSinglePhoto(IMAGE_URI, PHOTO_ID);
    when(driveItemsResponse.getDriveItems()).thenReturn(new MicrosoftDriveItem[] { photoItem });
    when(driveItemsResponse.getNextPageLink()).thenReturn(DRIVE_PAGE_URL);
    IdOnlyContainerResource idOnlyContainerResource = new IdOnlyContainerResource(FOLDER_ID);
    // Run
    ExportResult<PhotosContainerResource> result = microsoftPhotosExporter.exportOneDrivePhotos(null, Optional.of(idOnlyContainerResource), Optional.empty(), uuid);
    // Verify method calls
    verify(photosInterface).getDriveItems(Optional.of(FOLDER_ID), Optional.empty());
    verify(driveItemsResponse).getDriveItems();
    // Verify pagination token is set
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(DRIVE_TOKEN_PREFIX + DRIVE_PAGE_URL);
    // Verify no albums are exported
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums).isEmpty();
    // Verify one photo (in an album) should be exported
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos.stream().map(PhotoModel::getFetchableUrl).collect(Collectors.toList())).containsExactly(IMAGE_URI);
    assertThat(actualPhotos.stream().map(PhotoModel::getAlbumId).collect(Collectors.toList())).containsExactly(FOLDER_ID);
    assertThat(actualPhotos.stream().map(PhotoModel::getTitle).collect(Collectors.toList())).containsExactly(FILENAME);
    // Verify there are no containers ready for sub-processing
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources).isEmpty();
}
Also used : PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) PhotosContainerResource(org.datatransferproject.types.common.models.photos.PhotosContainerResource) ContainerResource(org.datatransferproject.types.common.models.ContainerResource) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) PhotoModel(org.datatransferproject.types.common.models.photos.PhotoModel) IdOnlyContainerResource(org.datatransferproject.types.common.models.IdOnlyContainerResource) ContinuationData(org.datatransferproject.spi.transfer.types.ContinuationData) PhotoAlbum(org.datatransferproject.types.common.models.photos.PhotoAlbum) StringPaginationToken(org.datatransferproject.types.common.StringPaginationToken) Test(org.junit.Test)

Aggregations

ContainerResource (org.datatransferproject.types.common.models.ContainerResource)24 Test (org.junit.Test)20 ContinuationData (org.datatransferproject.spi.transfer.types.ContinuationData)12 ExportInformation (org.datatransferproject.types.common.ExportInformation)11 StringPaginationToken (org.datatransferproject.types.common.StringPaginationToken)10 IdOnlyContainerResource (org.datatransferproject.types.common.models.IdOnlyContainerResource)10 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)7 PhotoAlbum (org.datatransferproject.types.common.models.photos.PhotoAlbum)7 InOrder (org.mockito.InOrder)7 List (java.util.List)6 Collectors (java.util.stream.Collectors)6 PaginationData (org.datatransferproject.types.common.PaginationData)6 PhotosContainerResource (org.datatransferproject.types.common.models.photos.PhotosContainerResource)6 Truth.assertThat (com.google.common.truth.Truth.assertThat)5 IOException (java.io.IOException)5 Collection (java.util.Collection)5 Optional (java.util.Optional)5 UUID (java.util.UUID)5 ExportResult (org.datatransferproject.spi.transfer.provider.ExportResult)5 PhotoModel (org.datatransferproject.types.common.models.photos.PhotoModel)5