Examples with ContinuationData org.datatransferproject.spi.transfer.types.ContinuationData used on opensource projects

Example 11 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class PortabilityInMemoryDataCopier method copyHelper.

/**
 * Transfers data from the given {@code exporter} optionally starting at the point specified in
 * the provided {@code exportInfo}. Imports the data using the provided {@code importer}. If there
 * is more data to required to be exported, recursively copies using the specific {@link
 * ExportInformation} to continue the process.
 *
 * @param exportAuthData The auth data for the export
 * @param importAuthData The auth data for the import
 * @param exportInfo Any pagination or resource information to use for subsequent calls.
 */
private Collection<ErrorDetail> copyHelper(AuthData exportAuthData, AuthData importAuthData, UUID jobId, Optional<ExportInformation> exportInfo) throws CopyException {
    String jobIdPrefix = "Job " + jobId + ": ";
    final int copyIteration = COPY_ITERATION_COUNTER.incrementAndGet();
    // NOTE: order is important below, do the import of all the items, then do continuation
    // then do sub resources, this ensures all parents are populated before children get
    // processed.
    ExportResult<?> exportResult = copyIteration(jobId, exportAuthData, importAuthData, exportInfo, jobIdPrefix, copyIteration);
    // Import and Export were successful, determine what to do next
    ContinuationData continuationData = exportResult.getContinuationData();
    if (null != continuationData) {
        // Process the next page of items for the resource
        if (null != continuationData.getPaginationData()) {
            monitor.debug(() -> jobIdPrefix + "Starting off a new copy iteration with pagination info, copy iteration: " + copyIteration);
            copyHelper(exportAuthData, importAuthData, jobId, Optional.of(new ExportInformation(continuationData.getPaginationData(), exportInfo.isPresent() ? exportInfo.get().getContainerResource() : null)));
        }
        // Start processing sub-resources
        if (continuationData.getContainerResources() != null && !continuationData.getContainerResources().isEmpty()) {
            for (ContainerResource resource : continuationData.getContainerResources()) {
                monitor.debug(() -> jobIdPrefix + "Starting off a new copy iteration with a new container resource, copy iteration: " + copyIteration);
                copyHelper(exportAuthData, importAuthData, jobId, Optional.of(new ExportInformation(null, resource)));
            }
        }
    }
    return idempotentImportExecutor.getErrors();
}

Example 12 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleCalendarExporterTest method exportEventFirstSet.

@Test
public void exportEventFirstSet() throws IOException {
    setUpSingleEventResponse();
    // Looking at first page, with at least one page after it
    ContainerResource containerResource = new IdOnlyContainerResource(CALENDAR_ID);
    ExportInformation exportInformation = new ExportInformation(null, containerResource);
    eventListResponse.setNextPageToken(NEXT_TOKEN);
    // Run test
    ExportResult<CalendarContainerResource> result = googleCalendarExporter.export(UUID.randomUUID(), null, Optional.of(exportInformation));
    // Check results
    // Verify correct methods were called
    verify(calendarEvents).list(CALENDAR_ID);
    verify(eventListRequest).setMaxAttendees(MAX_ATTENDEES);
    verify(eventListRequest).execute();
    // Check events
    Collection<CalendarEventModel> actualEvents = result.getExportedData().getEvents();
    assertThat(actualEvents.stream().map(CalendarEventModel::getCalendarId).collect(Collectors.toList())).containsExactly(CALENDAR_ID);
    assertThat(actualEvents.stream().map(CalendarEventModel::getTitle).collect(Collectors.toList())).containsExactly(EVENT_DESCRIPTION);
    // Check pagination token
    ContinuationData continuationData = (ContinuationData) result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken.getToken()).isEqualTo(EVENT_TOKEN_PREFIX + NEXT_TOKEN);
}

Example 13 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleCalendarExporterTest method exportEventSubsequentSet.

@Test
public void exportEventSubsequentSet() throws IOException {
    setUpSingleEventResponse();
    // Looking at subsequent page, with no pages after it
    ContainerResource containerResource = new IdOnlyContainerResource(CALENDAR_ID);
    PaginationData paginationData = new StringPaginationToken(EVENT_TOKEN_PREFIX + NEXT_TOKEN);
    ExportInformation exportInformation = new ExportInformation(paginationData, containerResource);
    eventListResponse.setNextPageToken(null);
    // Run test
    ExportResult<CalendarContainerResource> result = googleCalendarExporter.export(UUID.randomUUID(), null, Optional.of(exportInformation));
    // Check results
    // Verify correct methods were called in order
    InOrder inOrder = Mockito.inOrder(eventListRequest);
    inOrder.verify(eventListRequest).setPageToken(NEXT_TOKEN);
    inOrder.verify(eventListRequest).execute();
    // Check pagination token
    ContinuationData continuationData = (ContinuationData) result.getContinuationData();
    StringPaginationToken paginationToken = (StringPaginationToken) continuationData.getPaginationData();
    assertThat(paginationToken).isNull();
}

Example 14 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleTasksExporter method getTasks.

private ExportResult<TaskContainerResource> getTasks(Tasks tasksService, IdOnlyContainerResource resource, Optional<PaginationData> paginationData) throws IOException {
    Tasks.TasksOperations.List query = tasksService.tasks().list(resource.getId()).setMaxResults(PAGE_SIZE);
    if (paginationData.isPresent()) {
        query.setPageToken(((StringPaginationToken) paginationData.get()).getToken());
    }
    com.google.api.services.tasks.model.Tasks result = query.execute();
    List<TaskModel> newTasks = result.getItems().stream().map(t -> new TaskModel(resource.getId(), t.getTitle(), t.getNotes(), t.getCompleted() != null ? Instant.ofEpochMilli(t.getCompleted().getValue()) : null, t.getDue() != null ? Instant.ofEpochMilli(t.getDue().getValue()) : null)).collect(Collectors.toList());
    PaginationData newPage = null;
    ResultType resultType = ResultType.END;
    if (result.getNextPageToken() != null) {
        newPage = new StringPaginationToken(result.getNextPageToken());
        resultType = ResultType.CONTINUE;
    }
    TaskContainerResource taskContainerResource = new TaskContainerResource(null, newTasks);
    return new ExportResult<>(resultType, taskContainerResource, new ContinuationData(newPage));
}

Example 15 with ContinuationData

use of org.datatransferproject.spi.transfer.types.ContinuationData in project data-transfer-project by google.

the class GoogleTasksExporter method getTasksList.

private ExportResult<TaskContainerResource> getTasksList(Tasks tasksService, Optional<PaginationData> paginationData) throws IOException {
    Tasks.Tasklists.List query = tasksService.tasklists().list().setMaxResults(PAGE_SIZE);
    if (paginationData.isPresent()) {
        query.setPageToken(((StringPaginationToken) paginationData.get()).getToken());
    }
    TaskLists result = query.execute();
    ImmutableList.Builder<TaskListModel> newTaskListsBuilder = ImmutableList.builder();
    ImmutableList.Builder<IdOnlyContainerResource> newResourcesBuilder = ImmutableList.builder();
    for (TaskList taskList : result.getItems()) {
        newTaskListsBuilder.add(new TaskListModel(taskList.getId(), taskList.getTitle()));
        newResourcesBuilder.add(new IdOnlyContainerResource(taskList.getId()));
    }
    PaginationData newPage = null;
    ResultType resultType = ResultType.END;
    if (result.getNextPageToken() != null) {
        newPage = new StringPaginationToken(result.getNextPageToken());
        resultType = ResultType.CONTINUE;
    }
    List<IdOnlyContainerResource> newResources = newResourcesBuilder.build();
    if (!newResources.isEmpty()) {
        resultType = ResultType.CONTINUE;
    }
    TaskContainerResource taskContainerResource = new TaskContainerResource(newTaskListsBuilder.build(), null);
    ContinuationData continuationData = new ContinuationData(newPage);
    newResourcesBuilder.build().forEach(continuationData::addContainerResource);
    return new ExportResult<>(resultType, taskContainerResource, continuationData);
}