Examples with JobStore org.datatransferproject.spi.cloud.storage.JobStore used on opensource projects

Example 6 with JobStore

use of org.datatransferproject.spi.cloud.storage.JobStore in project data-transfer-project by google.

the class WorkerMain method initialize.

public void initialize() {
    Monitor monitor = loadMonitor();
    SettingsExtension settingsExtension = getSettingsExtension();
    settingsExtension.initialize();
    WorkerExtensionContext extensionContext = new WorkerExtensionContext(settingsExtension, monitor);
    // TODO this should be moved into a service extension
    extensionContext.registerService(HttpTransport.class, new NetHttpTransport());
    extensionContext.registerService(OkHttpClient.class, new OkHttpClient.Builder().build());
    extensionContext.registerService(JsonFactory.class, new JacksonFactory());
    ServiceLoader.load(ServiceExtension.class).iterator().forEachRemaining(serviceExtension -> serviceExtension.initialize(extensionContext));
    // TODO: verify that this is the cloud extension that is specified in the configuration
    CloudExtension cloudExtension = getCloudExtension();
    cloudExtension.initialize(extensionContext);
    monitor.info(() -> "Using CloudExtension: " + cloudExtension.getClass().getName());
    JobStore jobStore = cloudExtension.getJobStore();
    extensionContext.registerService(JobStore.class, jobStore);
    extensionContext.registerService(TemporaryPerJobDataStore.class, jobStore);
    AppCredentialStore appCredentialStore = cloudExtension.getAppCredentialStore();
    extensionContext.registerService(AppCredentialStore.class, appCredentialStore);
    List<TransferExtension> transferExtensions = getTransferExtensions(monitor);
    // Load security extension and services
    SecurityExtension securityExtension = SecurityExtensionLoader.getSecurityExtension(extensionContext);
    monitor.info(() -> "Using SecurityExtension: " + securityExtension.getClass().getName());
    IdempotentImportExecutor idempotentImportExecutor = IdempotentImportExecutorLoader.load(extensionContext);
    monitor.info(() -> "Using IdempotentImportExecutor: " + idempotentImportExecutor.getClass().getName());
    // TODO: make configurable
    SymmetricKeyGenerator symmetricKeyGenerator = new AesSymmetricKeyGenerator(monitor);
    JobHooks jobHooks = loadJobHooks();
    Injector injector = null;
    try {
        injector = Guice.createInjector(new WorkerModule(extensionContext, cloudExtension, transferExtensions, securityExtension, idempotentImportExecutor, symmetricKeyGenerator, jobHooks));
    } catch (Exception e) {
        monitor.severe(() -> "Unable to initialize Guice in Worker", e);
        throw e;
    }
    worker = injector.getInstance(Worker.class);
    // Reset the JobMetadata in case set previously when running SingleVMMain
    JobMetadata.reset();
}

Example 7 with JobStore

use of org.datatransferproject.spi.cloud.storage.JobStore in project data-transfer-project by google.

the class GooglePhotosImporterTest method importPhotoInTempStoreFailure.

@Test
public void importPhotoInTempStoreFailure() throws Exception {
    PhotoModel photoModel = new PhotoModel(PHOTO_TITLE, IMG_URI, PHOTO_DESCRIPTION, JPEG_MEDIA_TYPE, "oldPhotoID1", OLD_ALBUM_ID, true);
    Mockito.when(googlePhotosInterface.uploadPhotoContent(any())).thenThrow(new IOException("Unit Testing"));
    JobStore jobStore = Mockito.mock(LocalJobStore.class);
    Mockito.when(jobStore.getStream(any(), any())).thenReturn(new TemporaryPerJobDataStore.InputStreamWrapper(new ByteArrayInputStream("TestingBytes".getBytes())));
    Mockito.doNothing().when(jobStore).removeData(any(), anyString());
    GooglePhotosImporter googlePhotosImporter = new GooglePhotosImporter(null, jobStore, null, null, googlePhotosInterface, null, null, 1.0);
    BatchMediaItemResponse batchMediaItemResponse = new BatchMediaItemResponse(new NewMediaItemResult[] { buildMediaItemResult("token1", Code.OK_VALUE) });
    Mockito.when(googlePhotosInterface.createPhotos(any(NewMediaItemUpload.class))).thenReturn(batchMediaItemResponse);
    UUID jobId = UUID.randomUUID();
    googlePhotosImporter.importPhotoBatch(jobId, Mockito.mock(TokensAndUrlAuthData.class), Lists.newArrayList(photoModel), executor, NEW_ALBUM_ID);
    Mockito.verify(jobStore, Mockito.times(0)).removeData(any(), anyString());
    Mockito.verify(jobStore, Mockito.times(1)).getStream(any(), anyString());
}

Example 8 with JobStore

use of org.datatransferproject.spi.cloud.storage.JobStore in project data-transfer-project by google.

the class GooglePhotosImporterTest method setUp.

@Before
public void setUp() throws IOException, InvalidTokenException, PermissionDeniedException {
    googlePhotosInterface = Mockito.mock(GooglePhotosInterface.class);
    monitor = Mockito.mock(Monitor.class);
    executor = new InMemoryIdempotentImportExecutor(monitor);
    Mockito.when(googlePhotosInterface.makePostRequest(anyString(), any(), any(), eq(NewMediaItemResult.class))).thenReturn(Mockito.mock(NewMediaItemResult.class));
    JobStore jobStore = new LocalJobStore();
    InputStream inputStream = Mockito.mock(InputStream.class);
    imageStreamProvider = Mockito.mock(ImageStreamProvider.class);
    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
    Mockito.when(imageStreamProvider.getConnection(anyString())).thenReturn(conn);
    Mockito.when(conn.getInputStream()).thenReturn(inputStream);
    Mockito.when(conn.getContentLengthLong()).thenReturn(32L);
    googlePhotosImporter = new GooglePhotosImporter(null, jobStore, null, null, googlePhotosInterface, imageStreamProvider, monitor, 1.0);
}

Example 9 with JobStore

use of org.datatransferproject.spi.cloud.storage.JobStore in project data-transfer-project by google.

the class GoogleTransferExtension method initialize.

@Override
public void initialize(ExtensionContext context) {
    // times.
    if (initialized)
        return;
    JobStore jobStore = context.getService(JobStore.class);
    HttpTransport httpTransport = context.getService(HttpTransport.class);
    JsonFactory jsonFactory = context.getService(JsonFactory.class);
    AppCredentials appCredentials;
    try {
        appCredentials = context.getService(AppCredentialStore.class).getAppCredentials("GOOGLE_KEY", "GOOGLE_SECRET");
    } catch (IOException e) {
        Monitor monitor = context.getMonitor();
        monitor.info(() -> "Unable to retrieve Google AppCredentials. Did you set GOOGLE_KEY and GOOGLE_SECRET?");
        return;
    }
    Monitor monitor = context.getMonitor();
    // Create the GoogleCredentialFactory with the given {@link AppCredentials}.
    GoogleCredentialFactory credentialFactory = new GoogleCredentialFactory(httpTransport, jsonFactory, appCredentials, monitor);
    ImmutableMap.Builder<String, Importer> importerBuilder = ImmutableMap.builder();
    importerBuilder.put("BLOBS", new DriveImporter(credentialFactory, jobStore, monitor));
    importerBuilder.put("CONTACTS", new GoogleContactsImporter(credentialFactory));
    importerBuilder.put("CALENDAR", new GoogleCalendarImporter(credentialFactory));
    importerBuilder.put("MAIL", new GoogleMailImporter(credentialFactory, monitor));
    importerBuilder.put("TASKS", new GoogleTasksImporter(credentialFactory));
    importerBuilder.put("PHOTOS", new GooglePhotosImporter(credentialFactory, jobStore, jsonFactory, monitor, context.getSetting("googleWritesPerSecond", 1.0)));
    importerBuilder.put("VIDEOS", new GoogleVideosImporter(appCredentials, jobStore, monitor));
    importerMap = importerBuilder.build();
    ImmutableMap.Builder<String, Exporter> exporterBuilder = ImmutableMap.builder();
    exporterBuilder.put("BLOBS", new DriveExporter(credentialFactory, jobStore, monitor));
    exporterBuilder.put("CONTACTS", new GoogleContactsExporter(credentialFactory));
    exporterBuilder.put("CALENDAR", new GoogleCalendarExporter(credentialFactory));
    exporterBuilder.put("MAIL", new GoogleMailExporter(credentialFactory));
    exporterBuilder.put("SOCIAL-POSTS", new GooglePlusExporter(credentialFactory));
    exporterBuilder.put("TASKS", new GoogleTasksExporter(credentialFactory, monitor));
    exporterBuilder.put("PHOTOS", new GooglePhotosExporter(credentialFactory, jobStore, jsonFactory, monitor));
    exporterBuilder.put("VIDEOS", new GoogleVideosExporter(credentialFactory, jsonFactory));
    exporterMap = exporterBuilder.build();
    initialized = true;
}