Examples with ContentQueueManager me.devsaki.hentoid.util.download.ContentQueueManager used on opensource projects

Example 1 with ContentQueueManager

use of me.devsaki.hentoid.util.download.ContentQueueManager in project Hentoid by avluis.

the class ContentDownloadWorker method watchProgress.

/**
 * Watch download progress
 * <p>
 * NB : download pause is managed at the Volley queue level (see RequestQueueManager.pauseQueue / startQueue)
 *
 * @param content Content to watch (1st book of the download queue)
 */
private void watchProgress(@NonNull Content content) {
    boolean isDone;
    int pagesOK = 0;
    int pagesKO = 0;
    long downloadedBytes = 0;
    boolean firstPageDownloaded = false;
    int deltaPages = 0;
    int nbDeltaZeroPages = 0;
    long networkBytes = 0;
    long deltaNetworkBytes;
    int nbDeltaLowNetwork = 0;
    List<ImageFile> images = content.getImageFiles();
    // Compute total downloadable pages; online (stream) pages do not count
    int totalPages = (null == images) ? 0 : (int) Stream.of(images).filter(i -> !i.getStatus().equals(StatusContent.ONLINE)).count();
    ContentQueueManager contentQueueManager = ContentQueueManager.getInstance();
    do {
        Map<StatusContent, ImmutablePair<Integer, Long>> statuses = dao.countProcessedImagesById(content.getId());
        ImmutablePair<Integer, Long> status = statuses.get(StatusContent.DOWNLOADED);
        // Measure idle time since last iteration
        if (status != null) {
            deltaPages = status.left - pagesOK;
            if (deltaPages == 0)
                nbDeltaZeroPages++;
            else {
                firstPageDownloaded = true;
                nbDeltaZeroPages = 0;
            }
            pagesOK = status.left;
            downloadedBytes = status.right;
        }
        status = statuses.get(StatusContent.ERROR);
        if (status != null)
            pagesKO = status.left;
        double downloadedMB = downloadedBytes / (1024.0 * 1024);
        int progress = pagesOK + pagesKO;
        isDone = progress == totalPages;
        Timber.d("Progress: OK:%d size:%dMB - KO:%d - Total:%d", pagesOK, (int) downloadedMB, pagesKO, totalPages);
        // Download speed and size estimation
        long networkBytesNow = NetworkHelper.getIncomingNetworkUsage(getApplicationContext());
        deltaNetworkBytes = networkBytesNow - networkBytes;
        if (deltaNetworkBytes < 1024 * LOW_NETWORK_THRESHOLD && firstPageDownloaded)
            // LOW_NETWORK_THRESHOLD KBps threshold once download has started
            nbDeltaLowNetwork++;
        else
            nbDeltaLowNetwork = 0;
        networkBytes = networkBytesNow;
        downloadSpeedCalculator.addSampleNow(networkBytes);
        int avgSpeedKbps = (int) downloadSpeedCalculator.getAvgSpeedKbps();
        Timber.d("deltaPages: %d / deltaNetworkBytes: %s", deltaPages, FileHelper.formatHumanReadableSize(deltaNetworkBytes, getApplicationContext().getResources()));
        Timber.d("nbDeltaZeroPages: %d / nbDeltaLowNetwork: %d", nbDeltaZeroPages, nbDeltaLowNetwork);
        // Idle = very low download speed _AND_ no new pages downloaded
        if (nbDeltaLowNetwork > IDLE_THRESHOLD && nbDeltaZeroPages > IDLE_THRESHOLD) {
            nbDeltaLowNetwork = 0;
            nbDeltaZeroPages = 0;
            Timber.d("Inactivity detected ====> restarting request queue");
            // requestQueueManager.restartRequestQueue();
            requestQueueManager.resetRequestQueue(getApplicationContext(), false);
        }
        double estimateBookSizeMB = -1;
        if (pagesOK > 3 && progress > 0 && totalPages > 0) {
            estimateBookSizeMB = downloadedMB / (progress * 1.0 / totalPages);
            Timber.v("Estimate book size calculated for wifi check : %s MB", estimateBookSizeMB);
        }
        notificationManager.notify(new DownloadProgressNotification(content.getTitle(), progress, totalPages, (int) downloadedMB, (int) estimateBookSizeMB, avgSpeedKbps));
        EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_PROGRESS, pagesOK, pagesKO, totalPages, downloadedBytes));
        // If the "skip large downloads on mobile data" is on, skip if needed
        if (Preferences.isDownloadLargeOnlyWifi() && (estimateBookSizeMB > Preferences.getDownloadLargeOnlyWifiThresholdMB() || totalPages > Preferences.getDownloadLargeOnlyWifiThresholdPages())) {
            @NetworkHelper.Connectivity int connectivity = NetworkHelper.getConnectivity(getApplicationContext());
            if (NetworkHelper.Connectivity.WIFI != connectivity) {
                // Move the book to the errors queue and signal it as skipped
                logErrorRecord(content.getId(), ErrorType.WIFI, content.getUrl(), "Book", "");
                moveToErrors(content.getId());
                EventBus.getDefault().post(new DownloadEvent(DownloadEvent.Type.EV_SKIP));
            }
        }
        // We're polling the DB because we can't observe LiveData from a background service
        Helper.pause(1000);
    } while (!isDone && !downloadInterrupted.get() && !contentQueueManager.isQueuePaused());
    if (contentQueueManager.isQueuePaused()) {
        Timber.d("Content download paused : %s [%s]", content.getTitle(), content.getId());
        if (downloadCanceled.get())
            notificationManager.cancel();
    } else {
        // NB : no need to supply the Content itself as it has not been updated during the loop
        completeDownload(content.getId(), content.getTitle(), pagesOK, pagesKO, downloadedBytes);
    }
}

Example 2 with ContentQueueManager

use of me.devsaki.hentoid.util.download.ContentQueueManager in project Hentoid by avluis.

the class ContentDownloadWorker method completeDownload.

/**
 * Completes the download of a book when all images have been processed
 * Then launches a new IntentService
 *
 * @param contentId Id of the Content to mark as downloaded
 */
private void completeDownload(final long contentId, @NonNull final String title, final int pagesOK, final int pagesKO, final long sizeDownloadedBytes) {
    ContentQueueManager contentQueueManager = ContentQueueManager.getInstance();
    // Get the latest value of Content
    Content content = dao.selectContent(contentId);
    if (null == content) {
        Timber.w("Content ID %s not found", contentId);
        return;
    }
    if (!downloadInterrupted.get()) {
        List<ImageFile> images = content.getImageFiles();
        if (null == images)
            images = Collections.emptyList();
        // Don't count the cover
        int nbImages = (int) Stream.of(images).filter(i -> !i.isCover()).count();
        boolean hasError = false;
        // Set error state if less pages than initially detected - More than 10% difference in number of pages
        if (content.getQtyPages() > 0 && nbImages < content.getQtyPages() && Math.abs(nbImages - content.getQtyPages()) > content.getQtyPages() * 0.1) {
            String errorMsg = String.format("The number of images found (%s) does not match the book's number of pages (%s)", nbImages, content.getQtyPages());
            logErrorRecord(contentId, ErrorType.PARSING, content.getGalleryUrl(), "pages", errorMsg);
            hasError = true;
        }
        // Set error state if there are non-downloaded pages
        // NB : this should not happen theoretically
        long nbDownloadedPages = content.getNbDownloadedPages();
        if (nbDownloadedPages < content.getQtyPages()) {
            Timber.i(">> downloaded vs. qty KO %s vs %s", nbDownloadedPages, content.getQtyPages());
            String errorMsg = String.format("The number of downloaded images (%s) does not match the book's number of pages (%s)", nbDownloadedPages, content.getQtyPages());
            logErrorRecord(contentId, ErrorType.PARSING, content.getGalleryUrl(), "pages", errorMsg);
            hasError = true;
        }
        // update the book's number of pages and download date
        if (nbImages > content.getQtyPages()) {
            content.setQtyPages(nbImages);
            content.setDownloadDate(Instant.now().toEpochMilli());
        }
        if (content.getStorageUri().isEmpty())
            return;
        DocumentFile dir = FileHelper.getFolderFromTreeUriString(getApplicationContext(), content.getStorageUri());
        if (dir != null) {
            // TODO - test to make sure the service's thread continues to run in such a scenario
            if (pagesKO > 0 && Preferences.isDlRetriesActive() && content.getNumberDownloadRetries() < Preferences.getDlRetriesNumber()) {
                double freeSpaceRatio = new FileHelper.MemoryUsageFigures(getApplicationContext(), dir).getFreeUsageRatio100();
                if (freeSpaceRatio < Preferences.getDlRetriesMemLimit()) {
                    Timber.i("Initiating auto-retry #%s for content %s (%s%% free space)", content.getNumberDownloadRetries() + 1, content.getTitle(), freeSpaceRatio);
                    logErrorRecord(content.getId(), ErrorType.UNDEFINED, "", content.getTitle(), "Auto-retry #" + content.getNumberDownloadRetries());
                    content.increaseNumberDownloadRetries();
                    // Re-queue all failed images
                    for (ImageFile img : images) if (img.getStatus().equals(StatusContent.ERROR)) {
                        Timber.i("Auto-retry #%s for content %s / image @ %s", content.getNumberDownloadRetries(), content.getTitle(), img.getUrl());
                        img.setStatus(StatusContent.SAVED);
                        dao.insertImageFile(img);
                        requestQueueManager.queueRequest(buildImageDownloadRequest(img, dir, content));
                    }
                    return;
                }
            }
            // Compute perceptual hash for the cover picture
            ContentHelper.computeAndSaveCoverHash(getApplicationContext(), content, dao);
            // Mark content as downloaded
            if (0 == content.getDownloadDate())
                content.setDownloadDate(Instant.now().toEpochMilli());
            content.setStatus((0 == pagesKO && !hasError) ? StatusContent.DOWNLOADED : StatusContent.ERROR);
            // Clear download params from content
            if (0 == pagesKO && !hasError)
                content.setDownloadParams("");
            content.computeSize();
            // Save JSON file
            try {
                DocumentFile jsonFile = JsonHelper.jsonToFile(getApplicationContext(), JsonContent.fromEntity(content), JsonContent.class, dir);
                // Cache its URI to the newly created content
                if (jsonFile != null) {
                    content.setJsonUri(jsonFile.getUri().toString());
                } else {
                    Timber.w("JSON file could not be cached for %s", title);
                }
            } catch (IOException e) {
                Timber.e(e, "I/O Error saving JSON: %s", title);
            }
            ContentHelper.addContent(getApplicationContext(), dao, content);
            Timber.i("Content download finished: %s [%s]", title, contentId);
            // Delete book from queue
            dao.deleteQueue(content);
            // Increase downloads count
            contentQueueManager.downloadComplete();
            if (0 == pagesKO) {
                int downloadCount = contentQueueManager.getDownloadCount();
                notificationManager.notify(new DownloadSuccessNotification(downloadCount));
                // Tracking Event (Download Success)
                HentoidApp.trackDownloadEvent("Success");
            } else {
                notificationManager.notify(new DownloadErrorNotification(content));
                // Tracking Event (Download Error)
                HentoidApp.trackDownloadEvent("Error");
            }
            // Signals current download as completed
            Timber.d("CompleteActivity : OK = %s; KO = %s", pagesOK, pagesKO);
            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_COMPLETE, pagesOK, pagesKO, nbImages, sizeDownloadedBytes));
            Context context = getApplicationContext();
            if (ContentHelper.updateQueueJson(context, dao))
                Timber.i(context.getString(R.string.queue_json_saved));
            else
                Timber.w(context.getString(R.string.queue_json_failed));
            // Tracking Event (Download Completed)
            HentoidApp.trackDownloadEvent("Completed");
        } else {
            Timber.w("completeDownload : Directory %s does not exist - JSON not saved", content.getStorageUri());
        }
    } else if (downloadCanceled.get()) {
        Timber.d("Content download canceled: %s [%s]", title, contentId);
        notificationManager.cancel();
    } else {
        Timber.d("Content download skipped : %s [%s]", title, contentId);
    }
}