Examples with VideoModel org.datatransferproject.types.common.models.videos.VideoModel used on opensource projects

Example 11 with VideoModel

use of org.datatransferproject.types.common.models.videos.VideoModel in project data-transfer-project by google.

the class KoofrVideosImporterTest method testImportItemFromURLWithoutAlbum.

@Test
public void testImportItemFromURLWithoutAlbum() throws Exception {
    server.enqueue(new MockResponse().setResponseCode(200).setBody("123"));
    server.enqueue(new MockResponse().setResponseCode(200).setBody("4567"));
    when(client.ensureVideosFolder()).thenReturn("/root/Videos");
    UUID jobId = UUID.randomUUID();
    Collection<VideoAlbum> albums = ImmutableList.of();
    Collection<VideoModel> videos = ImmutableList.of(new VideoModel("video1.mp4", server.url("/1.mp4").toString(), "A video 1", "video/mp4", "video1", null, false), new VideoModel("video2.mp4", server.url("/2.mp4").toString(), "A video 2", "video/mp4", "video2", null, false));
    VideosContainerResource resource = spy(new VideosContainerResource(albums, videos));
    importer.importItem(jobId, executor, authData, resource);
    InOrder clientInOrder = Mockito.inOrder(client);
    clientInOrder.verify(client).ensureVideosFolder();
    clientInOrder.verify(client).fileExists(eq("/root/Videos/video1.mp4"));
    clientInOrder.verify(client).uploadFile(eq("/root/Videos"), eq("video1.mp4"), any(), eq("video/mp4"), isNull(), eq("A video 1"));
    clientInOrder.verify(client).fileExists(eq("/root/Videos/video2.mp4"));
    clientInOrder.verify(client).uploadFile(eq("/root/Videos"), eq("video2.mp4"), any(), eq("video/mp4"), isNull(), eq("A video 2"));
    clientInOrder.verifyNoMoreInteractions();
}

Example 12 with VideoModel

use of org.datatransferproject.types.common.models.videos.VideoModel in project data-transfer-project by google.

the class KoofrVideosImporterTest method testSkipNotFoundVideo.

@Test
public void testSkipNotFoundVideo() throws Exception {
    server.enqueue(new MockResponse().setResponseCode(404).setBody("4567"));
    UUID jobId = UUID.randomUUID();
    Collection<VideoAlbum> albums = ImmutableList.of();
    Collection<VideoModel> videos = ImmutableList.of(new VideoModel("not_found_video_1.mp4", server.url("/not_found.mp4").toString(), "Video not founded in CDN", "video/mp4", "not_found_video_1", null, false));
    VideosContainerResource resource = spy(new VideosContainerResource(albums, videos));
    importer.importItem(jobId, executor, authData, resource);
    InOrder clientInOrder = Mockito.inOrder(client);
    clientInOrder.verifyNoMoreInteractions();
}

Example 13 with VideoModel

use of org.datatransferproject.types.common.models.videos.VideoModel in project data-transfer-project by google.

the class GoogleVideosExporter method exportVideos.

@VisibleForTesting
ExportResult<VideosContainerResource> exportVideos(TokensAndUrlAuthData authData, Optional<StringPaginationToken> paginationData) throws IOException {
    Optional<String> paginationToken = paginationData.map(StringPaginationToken::getToken);
    MediaItemSearchResponse mediaItemSearchResponse = getOrCreateVideosInterface(authData).listVideoItems(paginationToken);
    PaginationData nextPageData = null;
    if (!Strings.isNullOrEmpty(mediaItemSearchResponse.getNextPageToken())) {
        nextPageData = new StringPaginationToken(mediaItemSearchResponse.getNextPageToken());
    }
    ContinuationData continuationData = new ContinuationData(nextPageData);
    VideosContainerResource containerResource = null;
    GoogleMediaItem[] mediaItems = mediaItemSearchResponse.getMediaItems();
    if (mediaItems != null && mediaItems.length > 0) {
        List<VideoModel> videos = convertVideosList(mediaItems);
        containerResource = new VideosContainerResource(null, videos);
    }
    ResultType resultType = ResultType.CONTINUE;
    if (nextPageData == null) {
        resultType = ResultType.END;
    }
    return new ExportResult<>(resultType, containerResource, continuationData);
}

Example 14 with VideoModel

use of org.datatransferproject.types.common.models.videos.VideoModel in project data-transfer-project by google.

the class GoogleVideosImporter method importItem.

@Override
public ImportResult importItem(UUID jobId, IdempotentImportExecutor executor, TokensAndUrlAuthData authData, VideosContainerResource data) throws Exception {
    if (data == null) {
        // Nothing to do
        return ImportResult.OK;
    }
    PhotosLibraryClient client;
    if (clientsMap.containsKey(jobId)) {
        client = clientsMap.get(jobId);
    } else {
        PhotosLibrarySettings settings = PhotosLibrarySettings.newBuilder().setCredentialsProvider(FixedCredentialsProvider.create(UserCredentials.newBuilder().setClientId(appCredentials.getKey()).setClientSecret(appCredentials.getSecret()).setAccessToken(new AccessToken(authData.getAccessToken(), new Date())).setRefreshToken(authData.getRefreshToken()).build())).build();
        client = PhotosLibraryClient.initialize(settings);
        clientsMap.put(jobId, client);
    }
    long bytes = 0L;
    // Uploads videos
    final Collection<VideoModel> videos = data.getVideos();
    if (videos != null && videos.size() > 0) {
        Stream<VideoModel> stream = videos.stream().filter(video -> shouldImport(video, executor)).map(this::transformVideoName);
        // We partition into groups of 49 as 50 is the maximum number of items that can be created in
        // one call. (We use 49 to avoid potential off by one errors)
        // https://developers.google.com/photos/library/guides/upload-media#creating-media-item
        final UnmodifiableIterator<List<VideoModel>> batches = Iterators.partition(stream.iterator(), 49);
        while (batches.hasNext()) {
            long batchBytes = importVideoBatch(batches.next(), client, executor);
            bytes += batchBytes;
        }
    }
    final ImportResult result = ImportResult.OK;
    return result.copyWithBytes(bytes);
}

Example 15 with VideoModel

use of org.datatransferproject.types.common.models.videos.VideoModel in project data-transfer-project by google.

the class GoogleVideosImporter method importVideoBatch.

long importVideoBatch(List<VideoModel> batchedVideos, PhotosLibraryClient client, IdempotentImportExecutor executor) throws Exception {
    final ArrayList<NewMediaItem> mediaItems = new ArrayList<>();
    final HashMap<String, VideoModel> uploadTokenToDataId = new HashMap<>();
    final HashMap<String, Long> uploadTokenToLength = new HashMap<>();
    // calls of the client to handle the InvalidArgumentException when the user's storage is full.
    try {
        for (VideoModel video : batchedVideos) {
            try {
                Pair<String, Long> pair = uploadMediaItem(video, client);
                final String uploadToken = pair.getLeft();
                mediaItems.add(buildMediaItem(video, uploadToken));
                uploadTokenToDataId.put(uploadToken, video);
                uploadTokenToLength.put(uploadToken, pair.getRight());
            } catch (IOException e) {
                if (e instanceof FileNotFoundException) {
                    // If the video file is no longer available then skip the video. We see this in a small
                    // number of videos where the video has been deleted.
                    monitor.info(() -> String.format("Video resource was missing for id: %s", video.getDataId()), e);
                    continue;
                }
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw e;
                });
            }
        }
        if (mediaItems.isEmpty()) {
            // Either we were not passed in any videos or we failed upload on all of them.
            return 0L;
        }
        BatchCreateMediaItemsResponse response = client.batchCreateMediaItems(mediaItems);
        final List<NewMediaItemResult> resultsList = response.getNewMediaItemResultsList();
        long bytes = 0L;
        for (NewMediaItemResult result : resultsList) {
            String uploadToken = result.getUploadToken();
            Status status = result.getStatus();
            final VideoModel video = uploadTokenToDataId.get(uploadToken);
            Preconditions.checkNotNull(video);
            final int code = status.getCode();
            if (code == Code.OK_VALUE) {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> result.getMediaItem().getId());
                Long length = uploadTokenToLength.get(uploadToken);
                if (length != null) {
                    bytes += length;
                }
            } else {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw new IOException(String.format("Video item could not be created. Code: %d Message: %s", code, result.getStatus().getMessage()));
                });
            }
            uploadTokenToDataId.remove(uploadToken);
        }
        if (!uploadTokenToDataId.isEmpty()) {
            for (VideoModel video : uploadTokenToDataId.values()) {
                executor.executeAndSwallowIOExceptions(video.getDataId(), video.getName(), () -> {
                    throw new IOException("Video item was missing from results list.");
                });
            }
        }
        return bytes;
    } catch (InvalidArgumentException e) {
        if (e.getMessage().contains("The remaining storage in the user's account is not enough")) {
            throw new DestinationMemoryFullException("Google destination storage full", e);
        } else {
            throw e;
        }
    }
}