Examples with ImportResult org.datatransferproject.spi.transfer.provider.ImportResult used on opensource projects

Example 6 with ImportResult

use of org.datatransferproject.spi.transfer.provider.ImportResult in project data-transfer-project by google.

the class MicrosoftPhotosImporterTest method testImportItemPermissionDenied.

@Test(expected = PermissionDeniedException.class)
public void testImportItemPermissionDenied() throws Exception {
    List<PhotoAlbum> albums = ImmutableList.of(new PhotoAlbum("id1", "album1.", "This is a fake albumb"));
    PhotosContainerResource data = new PhotosContainerResource(albums, null);
    Call call = mock(Call.class);
    doReturn(call).when(client).newCall(argThat((Request r) -> r.url().toString().equals("https://www.baseurl.com/v1.0/me/drive/special/photos/children")));
    Response response = mock(Response.class);
    ResponseBody body = mock(ResponseBody.class);
    when(body.bytes()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"id1\"}").bytes());
    when(body.string()).thenReturn(ResponseBody.create(MediaType.parse("application/json"), "{\"id\": \"id1\"}").string());
    when(response.code()).thenReturn(403);
    when(response.message()).thenReturn("Access Denied");
    when(response.body()).thenReturn(body);
    when(call.execute()).thenReturn(response);
    ImportResult result = importer.importItem(uuid, executor, authData, data);
}

Example 7 with ImportResult

use of org.datatransferproject.spi.transfer.provider.ImportResult in project data-transfer-project by google.

the class PortabilityAbstractInMemoryDataCopier method copyIteration.

protected ExportResult<?> copyIteration(UUID jobId, AuthData exportAuthData, AuthData importAuthData, Optional<ExportInformation> exportInformation, String jobIdPrefix, int copyIteration) throws CopyException {
    monitor.debug(() -> jobIdPrefix + "Copy iteration: " + copyIteration);
    RetryStrategyLibrary retryStrategyLibrary = retryStrategyLibraryProvider.get();
    monitor.debug(() -> jobIdPrefix + "Starting export, copy iteration: " + copyIteration, EventCode.COPIER_STARTED_EXPORT);
    CallableExporter callableExporter = new CallableExporter(exporterProvider, jobId, exportAuthData, exportInformation, metricRecorder);
    RetryingCallable<ExportResult> retryingExporter = new RetryingCallable<>(callableExporter, retryStrategyLibrary, Clock.systemUTC(), monitor, JobMetadata.getDataType(), JobMetadata.getExportService());
    ExportResult<?> exportResult;
    boolean exportSuccess = false;
    Stopwatch exportStopwatch = Stopwatch.createStarted();
    try {
        exportResult = retryingExporter.call();
        exportSuccess = exportResult.getType() != ExportResult.ResultType.ERROR;
    } catch (RetryException | RuntimeException e) {
        if (e.getClass() == RetryException.class && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) {
            throw (CopyExceptionWithFailureReason) e.getCause();
        }
        throw new CopyException(jobIdPrefix + "Error happened during export", e);
    } finally {
        metricRecorder.exportPageFinished(JobMetadata.getDataType(), JobMetadata.getExportService(), exportSuccess, exportStopwatch.elapsed());
    }
    monitor.debug(() -> jobIdPrefix + "Finished export, copy iteration: " + copyIteration, EventCode.COPIER_FINISHED_EXPORT);
    if (exportResult.getExportedData() != null) {
        monitor.debug(() -> jobIdPrefix + "Starting import, copy iteration: " + copyIteration, EventCode.COPIER_STARTED_IMPORT);
        CallableImporter callableImporter = new CallableImporter(importerProvider, jobId, idempotentImportExecutor, importAuthData, exportResult.getExportedData(), metricRecorder);
        RetryingCallable<ImportResult> retryingImporter = new RetryingCallable<>(callableImporter, retryStrategyLibrary, Clock.systemUTC(), monitor, JobMetadata.getDataType(), JobMetadata.getImportService());
        boolean importSuccess = false;
        Stopwatch importStopwatch = Stopwatch.createStarted();
        try {
            ImportResult importResult = retryingImporter.call();
            importSuccess = importResult.getType() == ImportResult.ResultType.OK;
            if (importSuccess) {
                try {
                    jobStore.addCounts(jobId, importResult.getCounts().orElse(null));
                    jobStore.addBytes(jobId, importResult.getBytes().orElse(null));
                } catch (IOException e) {
                    monitor.debug(() -> jobIdPrefix + "Unable to add counts to job: ", e);
                }
            }
        } catch (RetryException | RuntimeException e) {
            if (e.getClass() == RetryException.class && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) {
                throw (CopyExceptionWithFailureReason) e.getCause();
            }
            throw new CopyException(jobIdPrefix + "Error happened during import", e);
        } finally {
            metricRecorder.importPageFinished(JobMetadata.getDataType(), JobMetadata.getImportService(), importSuccess, importStopwatch.elapsed());
        }
        monitor.debug(() -> jobIdPrefix + "Finished import, copy iteration: " + copyIteration, EventCode.COPIER_FINISHED_IMPORT);
    }
    return exportResult;
}

Example 8 with ImportResult

use of org.datatransferproject.spi.transfer.provider.ImportResult in project data-transfer-project by google.

the class CallableImporter method call.

@Override
public ImportResult call() throws Exception {
    boolean success = false;
    Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        idempotentImportExecutor.resetRecentErrors();
        ImportResult result = importerProvider.get().importItem(jobId, idempotentImportExecutor, authData, data);
        Collection<ErrorDetail> errors = idempotentImportExecutor.getRecentErrors();
        success = result.getType() == ImportResult.ResultType.OK && errors.isEmpty();
        if (!success) {
            throw new IOException("Problem with importer, forcing a retry, " + "first error: " + (errors.iterator().hasNext() ? errors.iterator().next().exception() : "none"));
        }
        result = result.copyWithCounts(data.getCounts());
        return result;
    } finally {
        metricRecorder.importPageAttemptFinished(JobMetadata.getDataType(), JobMetadata.getImportService(), success, stopwatch.elapsed());
    }
}

Example 9 with ImportResult

use of org.datatransferproject.spi.transfer.provider.ImportResult in project data-transfer-project by google.

the class FlickrPhotosImporter method importItem.

@Override
public ImportResult importItem(UUID jobId, IdempotentImportExecutor idempotentExecutor, AuthData authData, PhotosContainerResource data) throws Exception, IOException {
    Auth auth;
    try {
        auth = FlickrUtils.getAuth(authData, flickr);
    } catch (FlickrException e) {
        return new ImportResult(e);
    }
    RequestContext.getRequestContext().setAuth(auth);
    Preconditions.checkArgument(data.getAlbums() != null || data.getPhotos() != null, "Error: There is no data to import");
    if (data.getAlbums() != null) {
        storeAlbums(jobId, data.getAlbums());
    }
    if (data.getPhotos() != null) {
        for (PhotoModel photo : data.getPhotos()) {
            try {
                importSinglePhoto(idempotentExecutor, jobId, photo);
            } catch (FlickrException e) {
                if (e.getMessage().contains("Upload limit reached")) {
                    throw new DestinationMemoryFullException("Flickr destination memory reached", e);
                } else if (e.getMessage().contains("Photo already in set")) {
                    // uploaded
                    continue;
                }
                throw new IOException(e);
            }
        }
    }
    return new ImportResult(ImportResult.ResultType.OK);
}

Example 10 with ImportResult

use of org.datatransferproject.spi.transfer.provider.ImportResult in project data-transfer-project by google.

the class FlickrPhotosImporterTest method importStoresAlbumInJobStore.

@Test
public void importStoresAlbumInJobStore() throws FlickrException, Exception {
    UUID jobId = UUID.randomUUID();
    PhotosContainerResource photosContainerResource = new PhotosContainerResource(Collections.singletonList(PHOTO_ALBUM), Collections.singletonList(PHOTO_MODEL));
    // Setup Mock
    when(user.getId()).thenReturn("userId");
    when(authInterface.checkToken(any(Token.class))).thenReturn(auth);
    when(flickr.getPhotosetsInterface()).thenReturn(photosetsInterface);
    when(flickr.getUploader()).thenReturn(uploader);
    when(flickr.getAuthInterface()).thenReturn(authInterface);
    when(imageStreamProvider.get(FETCHABLE_URL)).thenReturn(bufferedInputStream);
    when(uploader.upload(any(BufferedInputStream.class), any(UploadMetaData.class))).thenReturn(FLICKR_PHOTO_ID);
    String flickrAlbumTitle = ALBUM_NAME;
    Photoset photoset = FlickrTestUtils.initializePhotoset(FLICKR_ALBUM_ID, ALBUM_DESCRIPTION, FLICKR_PHOTO_ID);
    when(photosetsInterface.create(flickrAlbumTitle, ALBUM_DESCRIPTION, FLICKR_PHOTO_ID)).thenReturn(photoset);
    // Run test
    FlickrPhotosImporter importer = new FlickrPhotosImporter(flickr, jobStore, imageStreamProvider, monitor, TransferServiceConfig.getDefaultInstance());
    ImportResult result = importer.importItem(jobId, EXECUTOR, new TokenSecretAuthData("token", "secret"), photosContainerResource);
    // Verify that the image stream provider got the correct URL and that the correct info was
    // uploaded
    verify(imageStreamProvider).get(FETCHABLE_URL);
    ArgumentCaptor<UploadMetaData> uploadMetaDataArgumentCaptor = ArgumentCaptor.forClass(UploadMetaData.class);
    verify(uploader).upload(eq(bufferedInputStream), uploadMetaDataArgumentCaptor.capture());
    UploadMetaData actualUploadMetaData = uploadMetaDataArgumentCaptor.getValue();
    assertThat(actualUploadMetaData.getTitle()).isEqualTo(PHOTO_TITLE);
    assertThat(actualUploadMetaData.getDescription()).isEqualTo(PHOTO_DESCRIPTION);
    // Verify the photosets interface got the command to create the correct album
    verify(photosetsInterface).create(flickrAlbumTitle, ALBUM_DESCRIPTION, FLICKR_PHOTO_ID);
    assertThat((String) EXECUTOR.getCachedValue(ALBUM_ID)).isEqualTo(FLICKR_ALBUM_ID);
}