Examples with ContainerResource org.datatransferproject.types.common.models.ContainerResource used on opensource projects

Example 1 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class FacebookPhotosExporter method export.

@Override
public ExportResult<PhotosContainerResource> export(UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation) throws CopyExceptionWithFailureReason {
    Preconditions.checkNotNull(authData);
    if (!exportInformation.isPresent()) {
        // Start by getting the list of albums to export
        return exportAlbums(authData, Optional.empty());
    }
    StringPaginationToken paginationToken = (StringPaginationToken) exportInformation.get().getPaginationData();
    ContainerResource containerResource = exportInformation.get().getContainerResource();
    boolean containerResourcePresent = containerResource != null;
    boolean paginationDataPresent = paginationToken != null;
    if (!containerResourcePresent && paginationDataPresent && paginationToken.getToken().startsWith(ALBUM_TOKEN_PREFIX)) {
        // Continue exporting albums
        return exportAlbums(authData, Optional.of(paginationToken));
    } else if (containerResourcePresent && containerResource instanceof PhotosContainerResource) {
        // We have had albums specified from the front end so process them for import
        PhotosContainerResource photosContainerResource = (PhotosContainerResource) containerResource;
        Preconditions.checkNotNull(photosContainerResource.getAlbums());
        ContinuationData continuationData = new ContinuationData(null);
        for (PhotoAlbum album : photosContainerResource.getAlbums()) {
            continuationData.addContainerResource(new IdOnlyContainerResource(album.getId()));
        }
        return new ExportResult<>(ExportResult.ResultType.CONTINUE, photosContainerResource, continuationData);
    } else if (containerResourcePresent && containerResource instanceof IdOnlyContainerResource) {
        // Export photos
        return exportPhotos(jobId, authData, (IdOnlyContainerResource) containerResource, Optional.ofNullable(paginationToken));
    } else {
        throw new IllegalStateException(String.format("Invalid state passed into FacebookPhotosExporter. ExportInformation: %s", exportInformation));
    }
}

Example 2 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class PortabilityStackInMemoryDataCopier method updateStackAfterCopyIteration.

private void updateStackAfterCopyIteration(UUID jobId, String jobIdPrefix, ContainerResource exportContainerResource, int copyIteration, ContinuationData continuationData) {
    if (null != continuationData) {
        // Start processing sub-resources
        if (continuationData.getContainerResources() != null && !continuationData.getContainerResources().isEmpty()) {
            List<ContainerResource> subResources = continuationData.getContainerResources();
            for (int i = subResources.size() - 1; i >= 0; i--) {
                monitor.debug(() -> jobIdPrefix + "Pushing to the stack a new copy iteration with a new container resource, copy iteration: " + copyIteration);
                exportInfoStack.push((new ExportInformation(null, subResources.get(i))));
            }
        }
        // Push the next page of items onto the stack
        if (null != continuationData.getPaginationData()) {
            monitor.debug(() -> jobIdPrefix + "Pushing to the stack a new copy iteration with pagination info, copy iteration: " + copyIteration);
            exportInfoStack.push(new ExportInformation(continuationData.getPaginationData(), exportContainerResource));
        }
    }
    jobStore.storeJobStack(jobId, (Stack<ExportInformation>) exportInfoStack.clone());
}

Example 3 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class PortabilityStackInMemoryDataCopier method copy.

/**
 * Transfers data from the given {@code exporter} optionally starting at the point specified in
 * the provided {@code exportInformation}. Imports the data using the provided {@code importer}.
 * If there is more data to required to be exported, recursively copies using the specific {@link
 * ExportInformation} to continue the process.
 *
 * @param exportAuthData The auth data for the export
 * @param importAuthData The auth data for the import
 * @param exportInfo Any pagination or resource information to use for subsequent calls.
 */
@Override
public Collection<ErrorDetail> copy(AuthData exportAuthData, AuthData importAuthData, UUID jobId, Optional<ExportInformation> exportInfo) throws IOException, CopyException {
    idempotentImportExecutor.setJobId(jobId);
    String jobIdPrefix = "Job " + jobId + ": ";
    Optional<Stack<ExportInformation>> maybeLoadedStack = jobStore.loadJobStack(jobId);
    if (maybeLoadedStack.isPresent()) {
        // load stack from partially completed transfer
        exportInfoStack = maybeLoadedStack.get();
    } else {
        // start new transfer
        int initialCopyIteration = COPY_ITERATION_COUNTER.incrementAndGet();
        ExportResult<?> initialExportResult = copyIteration(jobId, exportAuthData, importAuthData, exportInfo, jobIdPrefix, initialCopyIteration);
        // Import and Export were successful, determine what to do next
        ContainerResource exportContainerResource = exportInfo.isPresent() ? exportInfo.get().getContainerResource() : null;
        updateStackAfterCopyIteration(jobId, jobIdPrefix, exportContainerResource, initialCopyIteration, initialExportResult.getContinuationData());
    }
    while (!exportInfoStack.isEmpty()) {
        int copyIteration = COPY_ITERATION_COUNTER.incrementAndGet();
        ExportInformation currentExportInfo = exportInfoStack.pop();
        ExportResult<?> exportResult = copyIteration(jobId, exportAuthData, importAuthData, Optional.of(currentExportInfo), jobIdPrefix, copyIteration);
        // Import and Export were successful, determine what to do next
        updateStackAfterCopyIteration(jobId, jobIdPrefix, currentExportInfo.getContainerResource(), copyIteration, exportResult.getContinuationData());
    }
    return idempotentImportExecutor.getErrors();
}

Example 4 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class MicrosoftPhotosExporterTest method exportOneAlbumWithNextPage.

@Test
public void exportOneAlbumWithNextPage() throws IOException {
    // Setup
    MicrosoftDriveItem folderItem = setUpSingleAlbum();
    when(driveItemsResponse.getDriveItems()).thenReturn(new MicrosoftDriveItem[] { folderItem });
    when(driveItemsResponse.getNextPageLink()).thenReturn(DRIVE_PAGE_URL);
    // Run
    ExportResult<PhotosContainerResource> result = microsoftPhotosExporter.exportOneDrivePhotos(null, Optional.empty(), Optional.empty(), uuid);
    // Verify method calls
    verify(photosInterface).getDriveItemsFromSpecialFolder(MicrosoftSpecialFolder.FolderType.photos);
    verify(driveItemsResponse).getDriveItems();
    // Verify pagination token is set
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(DRIVE_TOKEN_PREFIX + DRIVE_PAGE_URL);
    // Verify one album is ready for import
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums.stream().map(PhotoAlbum::getId).collect(Collectors.toList())).containsExactly(FOLDER_ID);
    // Verify photos should be empty (in the root)
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos).isEmpty();
    // Verify there is one container ready for sub-processing
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources.stream().map(a -> ((IdOnlyContainerResource) a).getId()).collect(Collectors.toList())).containsExactly(FOLDER_ID);
}

Example 5 with ContainerResource

use of org.datatransferproject.types.common.models.ContainerResource in project data-transfer-project by google.

the class MicrosoftPhotosExporterTest method exportPhotoWithNextPage.

@Test
public void exportPhotoWithNextPage() throws IOException {
    // Setup
    when(driveItemsResponse.getNextPageLink()).thenReturn(null);
    MicrosoftDriveItem photoItem = setUpSinglePhoto(IMAGE_URI, PHOTO_ID);
    when(driveItemsResponse.getDriveItems()).thenReturn(new MicrosoftDriveItem[] { photoItem });
    when(driveItemsResponse.getNextPageLink()).thenReturn(DRIVE_PAGE_URL);
    IdOnlyContainerResource idOnlyContainerResource = new IdOnlyContainerResource(FOLDER_ID);
    // Run
    ExportResult<PhotosContainerResource> result = microsoftPhotosExporter.exportOneDrivePhotos(null, Optional.of(idOnlyContainerResource), Optional.empty(), uuid);
    // Verify method calls
    verify(photosInterface).getDriveItems(Optional.of(FOLDER_ID), Optional.empty());
    verify(driveItemsResponse).getDriveItems();
    // Verify pagination token is set
    ContinuationData continuationData = result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(DRIVE_TOKEN_PREFIX + DRIVE_PAGE_URL);
    // Verify no albums are exported
    Collection<PhotoAlbum> actualAlbums = result.getExportedData().getAlbums();
    assertThat(actualAlbums).isEmpty();
    // Verify one photo (in an album) should be exported
    Collection<PhotoModel> actualPhotos = result.getExportedData().getPhotos();
    assertThat(actualPhotos.stream().map(PhotoModel::getFetchableUrl).collect(Collectors.toList())).containsExactly(IMAGE_URI);
    assertThat(actualPhotos.stream().map(PhotoModel::getAlbumId).collect(Collectors.toList())).containsExactly(FOLDER_ID);
    assertThat(actualPhotos.stream().map(PhotoModel::getTitle).collect(Collectors.toList())).containsExactly(FILENAME);
    // Verify there are no containers ready for sub-processing
    List<ContainerResource> actualResources = continuationData.getContainerResources();
    assertThat(actualResources).isEmpty();
}