Examples with IdempotentImportExecutor org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor used on opensource projects

Example 1 with IdempotentImportExecutor

use of org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor in project data-transfer-project by google.

the class GoogleBloggerImporter method insertActivity.

private void insertActivity(IdempotentImportExecutor idempotentExecutor, SocialActivityActor actor, SocialActivityModel activity, String blogId, TokensAndUrlAuthData authData) throws Exception {
    String content = activity.getContent() == null ? "" : activity.getContent();
    Collection<SocialActivityAttachment> linkAttachments = activity.getAttachments().stream().filter(attachment -> attachment.getType() == SocialActivityAttachmentType.LINK).collect(Collectors.toList());
    Collection<SocialActivityAttachment> imageAttachments = activity.getAttachments().stream().filter(attachment -> attachment.getType() == SocialActivityAttachmentType.IMAGE).collect(Collectors.toList());
    // don't know how they were laid out in the originating service.
    for (SocialActivityAttachment attachment : linkAttachments) {
        content = "<a href=\"" + attachment.getUrl() + "\">" + attachment.getName() + "</a>\n</hr>\n" + content;
    }
    if (!imageAttachments.isEmpty()) {
        // Store any attached images in Drive in a new folder.
        Drive driveInterface = getOrCreateDriveService(authData);
        String folderId = idempotentExecutor.executeOrThrowException("MainAlbum", "Photo Album", () -> createAlbumFolder(driveInterface));
        for (SocialActivityAttachment image : imageAttachments) {
            try {
                String newImgSrc = idempotentExecutor.executeAndSwallowIOExceptions(image.toString(), "Image", () -> uploadImage(image, driveInterface, folderId));
                content += "\n<hr/><img src=\"" + newImgSrc + "\">";
            } catch (RuntimeException e) {
                throw new IOException("Couldn't import: " + imageAttachments, e);
            }
        }
    }
    String title = "";
    if (activity.getTitle() != null && !Strings.isNullOrEmpty(activity.getTitle())) {
        title = activity.getTitle();
    }
    Post post = new Post().setTitle("Imported post: " + title).setContent(content);
    if (actor != null) {
        Post.Author author = new Post.Author();
        if (!Strings.isNullOrEmpty(actor.getName())) {
            author.setDisplayName(actor.getName());
        }
        if (!Strings.isNullOrEmpty(actor.getUrl())) {
            author.setUrl(actor.getUrl());
        }
        post.setAuthor(author);
    }
    if (activity.getPublished() != null) {
        post.setPublished(new DateTime(activity.getPublished().toEpochMilli()));
    }
    idempotentExecutor.executeAndSwallowIOExceptions(title, title, () -> getOrCreateBloggerService(authData).posts().insert(blogId, post).setIsDraft(true).execute().getId());
}

Example 2 with IdempotentImportExecutor

use of org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor in project data-transfer-project by google.

the class MicrosoftContactsImporter method importItem.

@Override
public ImportResult importItem(UUID jobId, IdempotentImportExecutor idempotentImportExecutor, TokenAuthData authData, ContactsModelWrapper wrapper) {
    JCardReader reader = new JCardReader(wrapper.getVCards());
    try {
        List<VCard> cards = reader.readAll();
        List<String> problems = new ArrayList<>();
        int[] id = new int[] { 1 };
        List<Map<String, Object>> requests = cards.stream().map(card -> {
            TransformResult<LinkedHashMap> result = transformerService.transform(LinkedHashMap.class, card);
            problems.addAll(result.getProblems());
            LinkedHashMap contact = result.getTransformed();
            Map<String, Object> request = createRequest(id[0], CONTACTS_URL, contact);
            id[0]++;
            return request;
        }).collect(toList());
        if (!problems.isEmpty()) {
        // TODO log problems
        }
        return batchRequest(authData, requests, baseUrl, client, objectMapper).getResult();
    } catch (IOException e) {
        // TODO log
        e.printStackTrace();
        return new ImportResult(e);
    }
}

Example 3 with IdempotentImportExecutor

use of org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor in project data-transfer-project by google.

the class WorkerMain method initialize.

public void initialize() {
    Monitor monitor = loadMonitor();
    SettingsExtension settingsExtension = getSettingsExtension();
    settingsExtension.initialize();
    WorkerExtensionContext extensionContext = new WorkerExtensionContext(settingsExtension, monitor);
    // TODO this should be moved into a service extension
    extensionContext.registerService(HttpTransport.class, new NetHttpTransport());
    extensionContext.registerService(OkHttpClient.class, new OkHttpClient.Builder().build());
    extensionContext.registerService(JsonFactory.class, new JacksonFactory());
    ServiceLoader.load(ServiceExtension.class).iterator().forEachRemaining(serviceExtension -> serviceExtension.initialize(extensionContext));
    // TODO: verify that this is the cloud extension that is specified in the configuration
    CloudExtension cloudExtension = getCloudExtension();
    cloudExtension.initialize(extensionContext);
    monitor.info(() -> "Using CloudExtension: " + cloudExtension.getClass().getName());
    JobStore jobStore = cloudExtension.getJobStore();
    extensionContext.registerService(JobStore.class, jobStore);
    extensionContext.registerService(TemporaryPerJobDataStore.class, jobStore);
    AppCredentialStore appCredentialStore = cloudExtension.getAppCredentialStore();
    extensionContext.registerService(AppCredentialStore.class, appCredentialStore);
    List<TransferExtension> transferExtensions = getTransferExtensions(monitor);
    // Load security extension and services
    SecurityExtension securityExtension = SecurityExtensionLoader.getSecurityExtension(extensionContext);
    monitor.info(() -> "Using SecurityExtension: " + securityExtension.getClass().getName());
    IdempotentImportExecutor idempotentImportExecutor = IdempotentImportExecutorLoader.load(extensionContext);
    monitor.info(() -> "Using IdempotentImportExecutor: " + idempotentImportExecutor.getClass().getName());
    // TODO: make configurable
    SymmetricKeyGenerator symmetricKeyGenerator = new AesSymmetricKeyGenerator(monitor);
    JobHooks jobHooks = loadJobHooks();
    Injector injector = null;
    try {
        injector = Guice.createInjector(new WorkerModule(extensionContext, cloudExtension, transferExtensions, securityExtension, idempotentImportExecutor, symmetricKeyGenerator, jobHooks));
    } catch (Exception e) {
        monitor.severe(() -> "Unable to initialize Guice in Worker", e);
        throw e;
    }
    worker = injector.getInstance(Worker.class);
    // Reset the JobMetadata in case set previously when running SingleVMMain
    JobMetadata.reset();
}

Example 4 with IdempotentImportExecutor

use of org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor in project data-transfer-project by google.

the class GoogleVideosImporter method importItem.

@Override
public ImportResult importItem(UUID jobId, IdempotentImportExecutor executor, TokensAndUrlAuthData authData, VideosContainerResource data) throws Exception {
    if (data == null) {
        // Nothing to do
        return ImportResult.OK;
    }
    PhotosLibraryClient client;
    if (clientsMap.containsKey(jobId)) {
        client = clientsMap.get(jobId);
    } else {
        PhotosLibrarySettings settings = PhotosLibrarySettings.newBuilder().setCredentialsProvider(FixedCredentialsProvider.create(UserCredentials.newBuilder().setClientId(appCredentials.getKey()).setClientSecret(appCredentials.getSecret()).setAccessToken(new AccessToken(authData.getAccessToken(), new Date())).setRefreshToken(authData.getRefreshToken()).build())).build();
        client = PhotosLibraryClient.initialize(settings);
        clientsMap.put(jobId, client);
    }
    long bytes = 0L;
    // Uploads videos
    final Collection<VideoModel> videos = data.getVideos();
    if (videos != null && videos.size() > 0) {
        Stream<VideoModel> stream = videos.stream().filter(video -> shouldImport(video, executor)).map(this::transformVideoName);
        // We partition into groups of 49 as 50 is the maximum number of items that can be created in
        // one call. (We use 49 to avoid potential off by one errors)
        // https://developers.google.com/photos/library/guides/upload-media#creating-media-item
        final UnmodifiableIterator<List<VideoModel>> batches = Iterators.partition(stream.iterator(), 49);
        while (batches.hasNext()) {
            long batchBytes = importVideoBatch(batches.next(), client, executor);
            bytes += batchBytes;
        }
    }
    final ImportResult result = ImportResult.OK;
    return result.copyWithBytes(bytes);
}

Example 5 with IdempotentImportExecutor

use of org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor in project data-transfer-project by google.

the class DaybookPostsImporter method insertActivity.

private String insertActivity(IdempotentImportExecutor executor, SocialActivityModel activity, TokensAndUrlAuthData authData) throws IOException {
    Map<String, String> imageMap = new HashMap<>();
    Map<String, String> linkMap = new HashMap<>();
    String content = activity.getContent() == null ? "" : activity.getContent();
    String title = activity.getTitle() == null ? "" : activity.getTitle();
    String location = activity.getLocation() == null || activity.getLocation().getName() == null ? "" : activity.getLocation().getName();
    String published = activity.getPublished().toString() == null ? "" : activity.getPublished().toString();
    Request.Builder requestBuilder = new Request.Builder().url(baseUrl);
    requestBuilder.header("token", authData.getAccessToken());
    FormBody.Builder builder = new FormBody.Builder().add("type", "POSTS");
    builder.add("exporter", JobMetadata.getExportService());
    builder.add("content", content);
    builder.add("title", title);
    builder.add("location", location);
    builder.add("published", published);
    Collection<SocialActivityAttachment> linkAttachments = activity.getAttachments().stream().filter(attachment -> attachment.getType() == SocialActivityAttachmentType.LINK).collect(Collectors.toList());
    Collection<SocialActivityAttachment> imageAttachments = activity.getAttachments().stream().filter(attachment -> attachment.getType() == SocialActivityAttachmentType.IMAGE).collect(Collectors.toList());
    // don't know how they were laid out in the originating service.
    if (!linkAttachments.isEmpty()) {
        for (SocialActivityAttachment attachment : linkAttachments) {
            linkMap.put(attachment.getName(), attachment.getUrl());
        }
        try {
            String json = objectMapper.writeValueAsString(linkMap);
            builder.add("link", json);
        } catch (JsonProcessingException e) {
            monitor.info(() -> String.format("Error processing JSON: %s", e.getMessage()));
        }
    }
    if (!imageAttachments.isEmpty()) {
        for (SocialActivityAttachment image : imageAttachments) {
            imageMap.put(image.getName() != null ? image.getName() : image.getUrl(), image.getUrl());
        }
        try {
            String json = objectMapper.writeValueAsString(imageMap);
            builder.add("image", json);
        } catch (JsonProcessingException e) {
            monitor.info(() -> String.format("Error processing JSON: %s", e.getMessage()));
        }
    }
    FormBody formBody = builder.build();
    requestBuilder.post(formBody);
    try (Response response = client.newCall(requestBuilder.build()).execute()) {
        int code = response.code();
        // Though sometimes it returns error code for success requests
        if (code < 200 || code > 299) {
            throw new IOException(String.format("Error occurred in request for adding entry, message: %s", response.message()));
        }
        return response.message();
    }
}