Examples with ExportResult org.datatransferproject.spi.transfer.provider.ExportResult used on opensource projects

Example 11 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class PortabilityAbstractInMemoryDataCopier method copyIteration.

protected ExportResult<?> copyIteration(UUID jobId, AuthData exportAuthData, AuthData importAuthData, Optional<ExportInformation> exportInformation, String jobIdPrefix, int copyIteration) throws CopyException {
    monitor.debug(() -> jobIdPrefix + "Copy iteration: " + copyIteration);
    RetryStrategyLibrary retryStrategyLibrary = retryStrategyLibraryProvider.get();
    monitor.debug(() -> jobIdPrefix + "Starting export, copy iteration: " + copyIteration, EventCode.COPIER_STARTED_EXPORT);
    CallableExporter callableExporter = new CallableExporter(exporterProvider, jobId, exportAuthData, exportInformation, metricRecorder);
    RetryingCallable<ExportResult> retryingExporter = new RetryingCallable<>(callableExporter, retryStrategyLibrary, Clock.systemUTC(), monitor, JobMetadata.getDataType(), JobMetadata.getExportService());
    ExportResult<?> exportResult;
    boolean exportSuccess = false;
    Stopwatch exportStopwatch = Stopwatch.createStarted();
    try {
        exportResult = retryingExporter.call();
        exportSuccess = exportResult.getType() != ExportResult.ResultType.ERROR;
    } catch (RetryException | RuntimeException e) {
        if (e.getClass() == RetryException.class && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) {
            throw (CopyExceptionWithFailureReason) e.getCause();
        }
        throw new CopyException(jobIdPrefix + "Error happened during export", e);
    } finally {
        metricRecorder.exportPageFinished(JobMetadata.getDataType(), JobMetadata.getExportService(), exportSuccess, exportStopwatch.elapsed());
    }
    monitor.debug(() -> jobIdPrefix + "Finished export, copy iteration: " + copyIteration, EventCode.COPIER_FINISHED_EXPORT);
    if (exportResult.getExportedData() != null) {
        monitor.debug(() -> jobIdPrefix + "Starting import, copy iteration: " + copyIteration, EventCode.COPIER_STARTED_IMPORT);
        CallableImporter callableImporter = new CallableImporter(importerProvider, jobId, idempotentImportExecutor, importAuthData, exportResult.getExportedData(), metricRecorder);
        RetryingCallable<ImportResult> retryingImporter = new RetryingCallable<>(callableImporter, retryStrategyLibrary, Clock.systemUTC(), monitor, JobMetadata.getDataType(), JobMetadata.getImportService());
        boolean importSuccess = false;
        Stopwatch importStopwatch = Stopwatch.createStarted();
        try {
            ImportResult importResult = retryingImporter.call();
            importSuccess = importResult.getType() == ImportResult.ResultType.OK;
            if (importSuccess) {
                try {
                    jobStore.addCounts(jobId, importResult.getCounts().orElse(null));
                    jobStore.addBytes(jobId, importResult.getBytes().orElse(null));
                } catch (IOException e) {
                    monitor.debug(() -> jobIdPrefix + "Unable to add counts to job: ", e);
                }
            }
        } catch (RetryException | RuntimeException e) {
            if (e.getClass() == RetryException.class && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) {
                throw (CopyExceptionWithFailureReason) e.getCause();
            }
            throw new CopyException(jobIdPrefix + "Error happened during import", e);
        } finally {
            metricRecorder.importPageFinished(JobMetadata.getDataType(), JobMetadata.getImportService(), importSuccess, importStopwatch.elapsed());
        }
        monitor.debug(() -> jobIdPrefix + "Finished import, copy iteration: " + copyIteration, EventCode.COPIER_FINISHED_IMPORT);
    }
    return exportResult;
}

Example 12 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class GoogleTasksExporter method getTasks.

private ExportResult<TaskContainerResource> getTasks(Tasks tasksService, IdOnlyContainerResource resource, Optional<PaginationData> paginationData) throws IOException {
    Tasks.TasksOperations.List query = tasksService.tasks().list(resource.getId()).setMaxResults(PAGE_SIZE);
    if (paginationData.isPresent()) {
        query.setPageToken(((StringPaginationToken) paginationData.get()).getToken());
    }
    com.google.api.services.tasks.model.Tasks result = query.execute();
    List<TaskModel> newTasks = result.getItems().stream().map(t -> new TaskModel(resource.getId(), t.getTitle(), t.getNotes(), t.getCompleted() != null ? Instant.ofEpochMilli(t.getCompleted().getValue()) : null, t.getDue() != null ? Instant.ofEpochMilli(t.getDue().getValue()) : null)).collect(Collectors.toList());
    PaginationData newPage = null;
    ResultType resultType = ResultType.END;
    if (result.getNextPageToken() != null) {
        newPage = new StringPaginationToken(result.getNextPageToken());
        resultType = ResultType.CONTINUE;
    }
    TaskContainerResource taskContainerResource = new TaskContainerResource(null, newTasks);
    return new ExportResult<>(resultType, taskContainerResource, new ContinuationData(newPage));
}

Example 13 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class GoogleTasksExporter method getTasksList.

private ExportResult<TaskContainerResource> getTasksList(Tasks tasksService, Optional<PaginationData> paginationData) throws IOException {
    Tasks.Tasklists.List query = tasksService.tasklists().list().setMaxResults(PAGE_SIZE);
    if (paginationData.isPresent()) {
        query.setPageToken(((StringPaginationToken) paginationData.get()).getToken());
    }
    TaskLists result = query.execute();
    ImmutableList.Builder<TaskListModel> newTaskListsBuilder = ImmutableList.builder();
    ImmutableList.Builder<IdOnlyContainerResource> newResourcesBuilder = ImmutableList.builder();
    for (TaskList taskList : result.getItems()) {
        newTaskListsBuilder.add(new TaskListModel(taskList.getId(), taskList.getTitle()));
        newResourcesBuilder.add(new IdOnlyContainerResource(taskList.getId()));
    }
    PaginationData newPage = null;
    ResultType resultType = ResultType.END;
    if (result.getNextPageToken() != null) {
        newPage = new StringPaginationToken(result.getNextPageToken());
        resultType = ResultType.CONTINUE;
    }
    List<IdOnlyContainerResource> newResources = newResourcesBuilder.build();
    if (!newResources.isEmpty()) {
        resultType = ResultType.CONTINUE;
    }
    TaskContainerResource taskContainerResource = new TaskContainerResource(newTaskListsBuilder.build(), null);
    ContinuationData continuationData = new ContinuationData(newPage);
    newResourcesBuilder.build().forEach(continuationData::addContainerResource);
    return new ExportResult<>(resultType, taskContainerResource, continuationData);
}

Example 14 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class FlickrPhotosExporter method exportPhotosContainer.

private ExportResult<PhotosContainerResource> exportPhotosContainer(PhotosContainerResource container) {
    ImmutableList.Builder<PhotoAlbum> albumBuilder = ImmutableList.builder();
    ImmutableList.Builder<PhotoModel> photosBuilder = ImmutableList.builder();
    List<IdOnlyContainerResource> subResources = new ArrayList<>();
    try {
        for (PhotoAlbum album : container.getAlbums()) {
            Photoset photoset = photosetsInterface.getInfo(album.getId());
            // Saving data to the album allows the target service to recreate the album structure
            albumBuilder.add(new PhotoAlbum(photoset.getId(), photoset.getTitle(), photoset.getDescription()));
            // Adding subresources tells the framework to recall export to get all the photos
            subResources.add(new IdOnlyContainerResource(photoset.getId()));
        }
        for (PhotoModel photo : container.getPhotos()) {
            Photo p = photosInterface.getInfo(photo.getDataId(), null);
            photosBuilder.add(toCommonPhoto(p, null));
        }
    } catch (FlickrException e) {
        return new ExportResult<>(e);
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(albumBuilder.build(), photosBuilder.build());
    ContinuationData continuationData = new ContinuationData(null);
    subResources.forEach(resource -> continuationData.addContainerResource(resource));
    return new ExportResult<>(ResultType.CONTINUE, photosContainerResource, continuationData);
}

Example 15 with ExportResult

use of org.datatransferproject.spi.transfer.provider.ExportResult in project data-transfer-project by google.

the class FlickrPhotosExporter method getPhotos.

private ExportResult<PhotosContainerResource> getPhotos(IdOnlyContainerResource resource, PaginationData paginationData) {
    String photoSetId = resource.getId();
    int page = paginationData == null ? 1 : ((IntPaginationToken) paginationData).getStart();
    PhotoList<Photo> photoSetList;
    try {
        if (Strings.isNullOrEmpty(photoSetId)) {
            RequestContext.getRequestContext().setExtras(EXTRAS);
            perUserRateLimiter.acquire();
            photoSetList = photosInterface.getNotInSet(PHOTO_PER_PAGE, page);
            RequestContext.getRequestContext().setExtras(ImmutableList.of());
        } else {
            perUserRateLimiter.acquire();
            photoSetList = photosetsInterface.getPhotos(photoSetId, ImmutableSet.copyOf(EXTRAS), 0, PHOTO_PER_PAGE, page);
        }
    } catch (FlickrException e) {
        return new ExportResult<>(e);
    }
    boolean hasMore = photoSetList.getPage() != photoSetList.getPages() && !photoSetList.isEmpty();
    Collection<PhotoModel> photos = photoSetList.stream().map(p -> toCommonPhoto(p, photoSetId)).collect(Collectors.toList());
    PaginationData newPage = null;
    if (hasMore) {
        newPage = new IntPaginationToken(page + 1);
    }
    // Get result type
    ResultType resultType = ResultType.CONTINUE;
    if (newPage == null) {
        resultType = ResultType.END;
    }
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(null, photos);
    return new ExportResult<>(resultType, photosContainerResource, new ContinuationData(newPage));
}