use of me.devsaki.hentoid.events.DownloadEvent in project Hentoid by avluis.
the class ContentDownloadWorker method watchProgress.
/**
* Watch download progress
* <p>
* NB : download pause is managed at the Volley queue level (see RequestQueueManager.pauseQueue / startQueue)
*
* @param content Content to watch (1st book of the download queue)
*/
private void watchProgress(@NonNull Content content) {
boolean isDone;
int pagesOK = 0;
int pagesKO = 0;
long downloadedBytes = 0;
boolean firstPageDownloaded = false;
int deltaPages = 0;
int nbDeltaZeroPages = 0;
long networkBytes = 0;
long deltaNetworkBytes;
int nbDeltaLowNetwork = 0;
List<ImageFile> images = content.getImageFiles();
// Compute total downloadable pages; online (stream) pages do not count
int totalPages = (null == images) ? 0 : (int) Stream.of(images).filter(i -> !i.getStatus().equals(StatusContent.ONLINE)).count();
ContentQueueManager contentQueueManager = ContentQueueManager.getInstance();
do {
Map<StatusContent, ImmutablePair<Integer, Long>> statuses = dao.countProcessedImagesById(content.getId());
ImmutablePair<Integer, Long> status = statuses.get(StatusContent.DOWNLOADED);
// Measure idle time since last iteration
if (status != null) {
deltaPages = status.left - pagesOK;
if (deltaPages == 0)
nbDeltaZeroPages++;
else {
firstPageDownloaded = true;
nbDeltaZeroPages = 0;
}
pagesOK = status.left;
downloadedBytes = status.right;
}
status = statuses.get(StatusContent.ERROR);
if (status != null)
pagesKO = status.left;
double downloadedMB = downloadedBytes / (1024.0 * 1024);
int progress = pagesOK + pagesKO;
isDone = progress == totalPages;
Timber.d("Progress: OK:%d size:%dMB - KO:%d - Total:%d", pagesOK, (int) downloadedMB, pagesKO, totalPages);
// Download speed and size estimation
long networkBytesNow = NetworkHelper.getIncomingNetworkUsage(getApplicationContext());
deltaNetworkBytes = networkBytesNow - networkBytes;
if (deltaNetworkBytes < 1024 * LOW_NETWORK_THRESHOLD && firstPageDownloaded)
// LOW_NETWORK_THRESHOLD KBps threshold once download has started
nbDeltaLowNetwork++;
else
nbDeltaLowNetwork = 0;
networkBytes = networkBytesNow;
downloadSpeedCalculator.addSampleNow(networkBytes);
int avgSpeedKbps = (int) downloadSpeedCalculator.getAvgSpeedKbps();
Timber.d("deltaPages: %d / deltaNetworkBytes: %s", deltaPages, FileHelper.formatHumanReadableSize(deltaNetworkBytes, getApplicationContext().getResources()));
Timber.d("nbDeltaZeroPages: %d / nbDeltaLowNetwork: %d", nbDeltaZeroPages, nbDeltaLowNetwork);
// Idle = very low download speed _AND_ no new pages downloaded
if (nbDeltaLowNetwork > IDLE_THRESHOLD && nbDeltaZeroPages > IDLE_THRESHOLD) {
nbDeltaLowNetwork = 0;
nbDeltaZeroPages = 0;
Timber.d("Inactivity detected ====> restarting request queue");
// requestQueueManager.restartRequestQueue();
requestQueueManager.resetRequestQueue(getApplicationContext(), false);
}
double estimateBookSizeMB = -1;
if (pagesOK > 3 && progress > 0 && totalPages > 0) {
estimateBookSizeMB = downloadedMB / (progress * 1.0 / totalPages);
Timber.v("Estimate book size calculated for wifi check : %s MB", estimateBookSizeMB);
}
notificationManager.notify(new DownloadProgressNotification(content.getTitle(), progress, totalPages, (int) downloadedMB, (int) estimateBookSizeMB, avgSpeedKbps));
EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_PROGRESS, pagesOK, pagesKO, totalPages, downloadedBytes));
// If the "skip large downloads on mobile data" is on, skip if needed
if (Preferences.isDownloadLargeOnlyWifi() && (estimateBookSizeMB > Preferences.getDownloadLargeOnlyWifiThresholdMB() || totalPages > Preferences.getDownloadLargeOnlyWifiThresholdPages())) {
@NetworkHelper.Connectivity int connectivity = NetworkHelper.getConnectivity(getApplicationContext());
if (NetworkHelper.Connectivity.WIFI != connectivity) {
// Move the book to the errors queue and signal it as skipped
logErrorRecord(content.getId(), ErrorType.WIFI, content.getUrl(), "Book", "");
moveToErrors(content.getId());
EventBus.getDefault().post(new DownloadEvent(DownloadEvent.Type.EV_SKIP));
}
}
// We're polling the DB because we can't observe LiveData from a background service
Helper.pause(1000);
} while (!isDone && !downloadInterrupted.get() && !contentQueueManager.isQueuePaused());
if (contentQueueManager.isQueuePaused()) {
Timber.d("Content download paused : %s [%s]", content.getTitle(), content.getId());
if (downloadCanceled.get())
notificationManager.cancel();
} else {
// NB : no need to supply the Content itself as it has not been updated during the loop
completeDownload(content.getId(), content.getTitle(), pagesOK, pagesKO, downloadedBytes);
}
}
use of me.devsaki.hentoid.events.DownloadEvent in project Hentoid by avluis.
the class ImportWorker method startImport.
/**
* Import books from known source folders
*
* @param rename True if the user has asked for a folder renaming when calling import from Preferences
* @param cleanNoJSON True if the user has asked for a cleanup of folders with no JSONs when calling import from Preferences
* @param cleanNoImages True if the user has asked for a cleanup of folders with no images when calling import from Preferences
*/
private void startImport(boolean rename, boolean cleanNoJSON, boolean cleanNoImages) {
booksOK = 0;
booksKO = 0;
nbFolders = 0;
List<LogHelper.LogEntry> log = new ArrayList<>();
Context context = getApplicationContext();
// Stop downloads; it can get messy if downloading _and_ refresh / import happen at the same time
EventBus.getDefault().post(new DownloadEvent(DownloadEvent.Type.EV_PAUSE));
DocumentFile rootFolder = FileHelper.getFolderFromTreeUriString(context, Preferences.getStorageUri());
if (null == rootFolder) {
Timber.e("Root folder is not defined (%s)", Preferences.getStorageUri());
return;
}
List<DocumentFile> bookFolders = new ArrayList<>();
CollectionDAO dao = new ObjectBoxDAO(context);
try (FileExplorer explorer = new FileExplorer(context, Uri.parse(Preferences.getStorageUri()))) {
// 1st pass : Import groups JSON
// Flag existing groups for cleanup
dao.flagAllGroups(Grouping.CUSTOM);
DocumentFile groupsFile = explorer.findFile(context, rootFolder, Consts.GROUPS_JSON_FILE_NAME);
if (groupsFile != null)
importGroups(context, groupsFile, dao, log);
else
trace(Log.INFO, STEP_GROUPS, log, "No groups file found");
// 2nd pass : count subfolders of every site folder
List<DocumentFile> siteFolders = explorer.listFolders(context, rootFolder);
int foldersProcessed = 1;
for (DocumentFile f : siteFolders) {
bookFolders.addAll(explorer.listFolders(context, f));
eventProgress(STEP_2_BOOK_FOLDERS, siteFolders.size(), foldersProcessed++, 0);
}
eventComplete(STEP_2_BOOK_FOLDERS, siteFolders.size(), siteFolders.size(), 0, null);
notificationManager.notify(new ImportProgressNotification(context.getResources().getString(R.string.starting_import), 0, 0));
// 3rd pass : scan every folder for a JSON file or subdirectories
String enabled = context.getResources().getString(R.string.enabled);
String disabled = context.getResources().getString(R.string.disabled);
trace(Log.DEBUG, 0, log, "Import books starting - initial detected count : %s", bookFolders.size() + "");
trace(Log.INFO, 0, log, "Rename folders %s", (rename ? enabled : disabled));
trace(Log.INFO, 0, log, "Remove folders with no JSONs %s", (cleanNoJSON ? enabled : disabled));
trace(Log.INFO, 0, log, "Remove folders with no images %s", (cleanNoImages ? enabled : disabled));
// Cleanup previously detected duplicates
DuplicatesDAO duplicatesDAO = new DuplicatesDAO(context);
try {
duplicatesDAO.clearEntries();
} finally {
duplicatesDAO.cleanup();
}
// Flag DB content for cleanup
dao.flagAllInternalBooks();
dao.flagAllErrorBooksWithJson();
for (int i = 0; i < bookFolders.size(); i++) {
if (isStopped())
throw new InterruptedException();
importFolder(context, explorer, dao, bookFolders, bookFolders.get(i), log, rename, cleanNoJSON, cleanNoImages);
}
trace(Log.INFO, STEP_3_BOOKS, log, "Import books complete - %s OK; %s KO; %s final count", booksOK + "", booksKO + "", bookFolders.size() - nbFolders + "");
eventComplete(STEP_3_BOOKS, bookFolders.size(), booksOK, booksKO, null);
// 4th pass : Import queue & bookmarks JSON
DocumentFile queueFile = explorer.findFile(context, rootFolder, Consts.QUEUE_JSON_FILE_NAME);
if (queueFile != null)
importQueue(context, queueFile, dao, log);
else
trace(Log.INFO, STEP_4_QUEUE_FINAL, log, "No queue file found");
DocumentFile bookmarksFile = explorer.findFile(context, rootFolder, Consts.BOOKMARKS_JSON_FILE_NAME);
if (bookmarksFile != null)
importBookmarks(context, bookmarksFile, dao, log);
else
trace(Log.INFO, STEP_4_QUEUE_FINAL, log, "No bookmarks file found");
} catch (IOException | InterruptedException e) {
Timber.w(e);
// Restore interrupted state
Thread.currentThread().interrupt();
} finally {
// Write log in root folder
DocumentFile logFile = LogHelper.writeLog(context, buildLogInfo(rename || cleanNoJSON || cleanNoImages, log));
dao.deleteAllFlaggedBooks(true);
dao.deleteAllFlaggedGroups();
dao.cleanup();
eventComplete(STEP_4_QUEUE_FINAL, bookFolders.size(), booksOK, booksKO, logFile);
notificationManager.notify(new ImportCompleteNotification(booksOK, booksKO));
}
}
use of me.devsaki.hentoid.events.DownloadEvent in project Hentoid by avluis.
the class QueueFragment method onCreateView.
@Override
public View onCreateView(@NonNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
rootView = inflater.inflate(R.layout.fragment_queue, container, false);
mEmptyText = requireViewById(rootView, R.id.queue_empty_txt);
btnStart = requireViewById(rootView, R.id.btnStart);
btnPause = requireViewById(rootView, R.id.btnPause);
queueStatus = requireViewById(rootView, R.id.queueStatus);
queueInfo = requireViewById(rootView, R.id.queueInfo);
dlPreparationProgressBar = requireViewById(rootView, R.id.queueDownloadPreparationProgressBar);
// Both queue control buttons actually just need to send a signal that will be processed accordingly by whom it may concern
btnStart.setOnClickListener(v -> EventBus.getDefault().post(new DownloadEvent(DownloadEvent.Type.EV_UNPAUSE)));
btnPause.setOnClickListener(v -> EventBus.getDefault().post(new DownloadEvent(DownloadEvent.Type.EV_PAUSE)));
// Book list
recyclerView = requireViewById(rootView, R.id.queue_list);
ContentItem item = new ContentItem(ContentItem.ViewType.QUEUE);
fastAdapter.registerItemFactory(item.getType(), item);
// Gets (or creates and attaches if not yet existing) the extension from the given `FastAdapter`
selectExtension = fastAdapter.getOrCreateExtension(SelectExtension.class);
if (selectExtension != null) {
selectExtension.setSelectable(true);
selectExtension.setMultiSelect(true);
selectExtension.setSelectOnLongClick(true);
selectExtension.setSelectWithItemUpdate(true);
selectExtension.setSelectionListener((i, b) -> this.onSelectionChanged());
FastAdapterPreClickSelectHelper<ContentItem> helper = new FastAdapterPreClickSelectHelper<>(selectExtension);
fastAdapter.setOnPreClickListener(helper::onPreClickListener);
fastAdapter.setOnPreLongClickListener(helper::onPreLongClickListener);
}
recyclerView.setAdapter(fastAdapter);
recyclerView.setHasFixedSize(true);
llm = (LinearLayoutManager) recyclerView.getLayoutManager();
// Fast scroller
new FastScrollerBuilder(recyclerView).build();
// Drag, drop & swiping
SimpleSwipeDrawerDragCallback dragSwipeCallback = new SimpleSwipeDrawerDragCallback(this, ItemTouchHelper.LEFT, this).withSwipeLeft(Helper.dimensAsDp(requireContext(), R.dimen.delete_drawer_width_list)).withSensitivity(1.5f).withSurfaceThreshold(0.3f).withNotifyAllDrops(true);
// Despite its name, that's actually to disable drag on long tap
dragSwipeCallback.setIsDragEnabled(false);
touchHelper = new ItemTouchHelper(dragSwipeCallback);
touchHelper.attachToRecyclerView(recyclerView);
// Item click listener
fastAdapter.setOnClickListener((v, a, i, p) -> onItemClick(i));
initToolbar();
initSelectionToolbar();
attachButtons(fastAdapter);
// Network usage display refresh
compositeDisposable.add(Observable.timer(1, TimeUnit.SECONDS).subscribeOn(Schedulers.computation()).repeat().observeOn(Schedulers.computation()).map(v -> NetworkHelper.getIncomingNetworkUsage(requireContext())).observeOn(AndroidSchedulers.mainThread()).subscribe(this::updateNetworkUsage));
addCustomBackControl();
return rootView;
}
use of me.devsaki.hentoid.events.DownloadEvent in project Hentoid by avluis.
the class ContentDownloadWorker method completeDownload.
/**
* Completes the download of a book when all images have been processed
* Then launches a new IntentService
*
* @param contentId Id of the Content to mark as downloaded
*/
private void completeDownload(final long contentId, @NonNull final String title, final int pagesOK, final int pagesKO, final long sizeDownloadedBytes) {
ContentQueueManager contentQueueManager = ContentQueueManager.getInstance();
// Get the latest value of Content
Content content = dao.selectContent(contentId);
if (null == content) {
Timber.w("Content ID %s not found", contentId);
return;
}
if (!downloadInterrupted.get()) {
List<ImageFile> images = content.getImageFiles();
if (null == images)
images = Collections.emptyList();
// Don't count the cover
int nbImages = (int) Stream.of(images).filter(i -> !i.isCover()).count();
boolean hasError = false;
// Set error state if less pages than initially detected - More than 10% difference in number of pages
if (content.getQtyPages() > 0 && nbImages < content.getQtyPages() && Math.abs(nbImages - content.getQtyPages()) > content.getQtyPages() * 0.1) {
String errorMsg = String.format("The number of images found (%s) does not match the book's number of pages (%s)", nbImages, content.getQtyPages());
logErrorRecord(contentId, ErrorType.PARSING, content.getGalleryUrl(), "pages", errorMsg);
hasError = true;
}
// Set error state if there are non-downloaded pages
// NB : this should not happen theoretically
long nbDownloadedPages = content.getNbDownloadedPages();
if (nbDownloadedPages < content.getQtyPages()) {
Timber.i(">> downloaded vs. qty KO %s vs %s", nbDownloadedPages, content.getQtyPages());
String errorMsg = String.format("The number of downloaded images (%s) does not match the book's number of pages (%s)", nbDownloadedPages, content.getQtyPages());
logErrorRecord(contentId, ErrorType.PARSING, content.getGalleryUrl(), "pages", errorMsg);
hasError = true;
}
// update the book's number of pages and download date
if (nbImages > content.getQtyPages()) {
content.setQtyPages(nbImages);
content.setDownloadDate(Instant.now().toEpochMilli());
}
if (content.getStorageUri().isEmpty())
return;
DocumentFile dir = FileHelper.getFolderFromTreeUriString(getApplicationContext(), content.getStorageUri());
if (dir != null) {
// TODO - test to make sure the service's thread continues to run in such a scenario
if (pagesKO > 0 && Preferences.isDlRetriesActive() && content.getNumberDownloadRetries() < Preferences.getDlRetriesNumber()) {
double freeSpaceRatio = new FileHelper.MemoryUsageFigures(getApplicationContext(), dir).getFreeUsageRatio100();
if (freeSpaceRatio < Preferences.getDlRetriesMemLimit()) {
Timber.i("Initiating auto-retry #%s for content %s (%s%% free space)", content.getNumberDownloadRetries() + 1, content.getTitle(), freeSpaceRatio);
logErrorRecord(content.getId(), ErrorType.UNDEFINED, "", content.getTitle(), "Auto-retry #" + content.getNumberDownloadRetries());
content.increaseNumberDownloadRetries();
// Re-queue all failed images
for (ImageFile img : images) if (img.getStatus().equals(StatusContent.ERROR)) {
Timber.i("Auto-retry #%s for content %s / image @ %s", content.getNumberDownloadRetries(), content.getTitle(), img.getUrl());
img.setStatus(StatusContent.SAVED);
dao.insertImageFile(img);
requestQueueManager.queueRequest(buildImageDownloadRequest(img, dir, content));
}
return;
}
}
// Compute perceptual hash for the cover picture
ContentHelper.computeAndSaveCoverHash(getApplicationContext(), content, dao);
// Mark content as downloaded
if (0 == content.getDownloadDate())
content.setDownloadDate(Instant.now().toEpochMilli());
content.setStatus((0 == pagesKO && !hasError) ? StatusContent.DOWNLOADED : StatusContent.ERROR);
// Clear download params from content
if (0 == pagesKO && !hasError)
content.setDownloadParams("");
content.computeSize();
// Save JSON file
try {
DocumentFile jsonFile = JsonHelper.jsonToFile(getApplicationContext(), JsonContent.fromEntity(content), JsonContent.class, dir);
// Cache its URI to the newly created content
if (jsonFile != null) {
content.setJsonUri(jsonFile.getUri().toString());
} else {
Timber.w("JSON file could not be cached for %s", title);
}
} catch (IOException e) {
Timber.e(e, "I/O Error saving JSON: %s", title);
}
ContentHelper.addContent(getApplicationContext(), dao, content);
Timber.i("Content download finished: %s [%s]", title, contentId);
// Delete book from queue
dao.deleteQueue(content);
// Increase downloads count
contentQueueManager.downloadComplete();
if (0 == pagesKO) {
int downloadCount = contentQueueManager.getDownloadCount();
notificationManager.notify(new DownloadSuccessNotification(downloadCount));
// Tracking Event (Download Success)
HentoidApp.trackDownloadEvent("Success");
} else {
notificationManager.notify(new DownloadErrorNotification(content));
// Tracking Event (Download Error)
HentoidApp.trackDownloadEvent("Error");
}
// Signals current download as completed
Timber.d("CompleteActivity : OK = %s; KO = %s", pagesOK, pagesKO);
EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_COMPLETE, pagesOK, pagesKO, nbImages, sizeDownloadedBytes));
Context context = getApplicationContext();
if (ContentHelper.updateQueueJson(context, dao))
Timber.i(context.getString(R.string.queue_json_saved));
else
Timber.w(context.getString(R.string.queue_json_failed));
// Tracking Event (Download Completed)
HentoidApp.trackDownloadEvent("Completed");
} else {
Timber.w("completeDownload : Directory %s does not exist - JSON not saved", content.getStorageUri());
}
} else if (downloadCanceled.get()) {
Timber.d("Content download canceled: %s [%s]", title, contentId);
notificationManager.cancel();
} else {
Timber.d("Content download skipped : %s [%s]", title, contentId);
}
}
use of me.devsaki.hentoid.events.DownloadEvent in project Hentoid by avluis.
the class ContentDownloadWorker method downloadFirstInQueue.
/**
* Start the download of the 1st book of the download queue
* <p>
* NB : This method is not only called the 1st time the queue is awakened,
* but also after every book has finished downloading
*
* @return 1st book of the download queue; null if no book is available to download
*/
@SuppressLint({ "TimberExceptionLogging", "TimberArgCount" })
@NonNull
private ImmutablePair<QueuingResult, Content> downloadFirstInQueue() {
final String CONTENT_PART_IMAGE_LIST = "Image list";
Context context = getApplicationContext();
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.INIT));
// Clear previously created requests
compositeDisposable.clear();
// Check if queue has been paused
if (ContentQueueManager.getInstance().isQueuePaused()) {
Timber.i("Queue is paused. Download aborted.");
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
@NetworkHelper.Connectivity int connectivity = NetworkHelper.getConnectivity(context);
// Check for network connectivity
if (NetworkHelper.Connectivity.NO_INTERNET == connectivity) {
Timber.i("No internet connection available. Queue paused.");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.NO_INTERNET));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
// Check for wifi if wifi-only mode is on
if (Preferences.isQueueWifiOnly() && NetworkHelper.Connectivity.WIFI != connectivity) {
Timber.i("No wi-fi connection available. Queue paused.");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.NO_WIFI));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
// Check for download folder existence, available free space and credentials
if (Preferences.getStorageUri().trim().isEmpty()) {
// May happen if user has skipped it during the intro
Timber.i("No download folder set");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.NO_DOWNLOAD_FOLDER));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
DocumentFile rootFolder = FileHelper.getFolderFromTreeUriString(context, Preferences.getStorageUri());
if (null == rootFolder) {
// May happen if the folder has been moved or deleted after it has been selected
Timber.i("Download folder has not been found. Please select it again.");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.DOWNLOAD_FOLDER_NOT_FOUND));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
if (!FileHelper.isUriPermissionPersisted(context.getContentResolver(), rootFolder.getUri())) {
Timber.i("Insufficient credentials on download folder. Please select it again.");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.DOWNLOAD_FOLDER_NO_CREDENTIALS));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
long spaceLeftBytes = new FileHelper.MemoryUsageFigures(context, rootFolder).getfreeUsageBytes();
if (spaceLeftBytes < 2L * 1024 * 1024) {
Timber.i("Device very low on storage space (<2 MB). Queue paused.");
EventBus.getDefault().post(DownloadEvent.fromPauseMotive(DownloadEvent.Motive.NO_STORAGE, spaceLeftBytes));
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
// Work on first item of queue
// Check if there is a first item to process
List<QueueRecord> queue = dao.selectQueue();
if (queue.isEmpty()) {
Timber.i("Queue is empty. Download aborted.");
return new ImmutablePair<>(QueuingResult.QUEUE_END, null);
}
Content content = queue.get(0).getContent().getTarget();
if (null == content) {
Timber.i("Content is unavailable. Download aborted.");
dao.deleteQueue(0);
// Must supply content ID to the event for the UI to update properly
content = new Content().setId(queue.get(0).getContent().getTargetId());
EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_COMPLETE, 0, 0, 0, 0));
notificationManager.notify(new DownloadErrorNotification());
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
}
if (StatusContent.DOWNLOADED == content.getStatus()) {
Timber.i("Content is already downloaded. Download aborted.");
dao.deleteQueue(0);
EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_COMPLETE, 0, 0, 0, 0));
notificationManager.notify(new DownloadErrorNotification(content));
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
}
downloadCanceled.set(false);
downloadSkipped.set(false);
downloadInterrupted.set(false);
isCloudFlareBlocked = false;
@Content.DownloadMode int downloadMode = content.getDownloadMode();
dao.deleteErrorRecords(content.getId());
// == PREPARATION PHASE ==
// Parse images from the site (using image list parser)
// - Case 1 : If no image is present => parse all images
// - Case 2 : If all images are in ERROR state => re-parse all images
// - Case 3 : If some images are in ERROR state and the site has backup URLs
// => re-parse images with ERROR state using their order as reference
boolean hasError = false;
int nbErrors = 0;
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.PROCESS_IMG));
List<ImageFile> images = content.getImageFiles();
if (null == images)
images = new ArrayList<>();
else
// Safe copy of the original list
images = new ArrayList<>(images);
for (ImageFile img : images) if (img.getStatus().equals(StatusContent.ERROR))
nbErrors++;
StatusContent targetImageStatus = (downloadMode == Content.DownloadMode.DOWNLOAD) ? StatusContent.SAVED : StatusContent.ONLINE;
if (images.isEmpty() || nbErrors == images.size() || (nbErrors > 0 && content.getSite().hasBackupURLs())) {
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.FETCH_IMG));
try {
List<ImageFile> newImages = ContentHelper.fetchImageURLs(content, targetImageStatus);
// Cases 1 and 2 : Replace existing images with the parsed images
if (images.isEmpty() || nbErrors == images.size())
images = newImages;
// Case 3 : Replace images in ERROR state with the parsed images at the same position
if (nbErrors > 0 && content.getSite().hasBackupURLs()) {
for (int i = 0; i < images.size(); i++) {
ImageFile oldImage = images.get(i);
if (oldImage.getStatus().equals(StatusContent.ERROR)) {
for (ImageFile newImg : newImages) if (newImg.getOrder().equals(oldImage.getOrder()))
images.set(i, newImg);
}
}
}
if (content.isUpdatedProperties())
dao.insertContent(content);
// Manually insert new images (without using insertContent)
dao.replaceImageList(content.getId(), images);
} catch (CaptchaException cpe) {
Timber.i(cpe, "A captcha has been found while parsing %s. Download aborted.", content.getTitle());
logErrorRecord(content.getId(), ErrorType.CAPTCHA, content.getUrl(), CONTENT_PART_IMAGE_LIST, "Captcha found. Please go back to the site, browse a book and solve the captcha.");
hasError = true;
} catch (AccountException ae) {
String description = String.format("Your %s account does not allow to download the book %s. %s. Download aborted.", content.getSite().getDescription(), content.getTitle(), ae.getMessage());
Timber.i(ae, description);
logErrorRecord(content.getId(), ErrorType.ACCOUNT, content.getUrl(), CONTENT_PART_IMAGE_LIST, description);
hasError = true;
} catch (LimitReachedException lre) {
String description = String.format("The bandwidth limit has been reached while parsing %s. %s. Download aborted.", content.getTitle(), lre.getMessage());
Timber.i(lre, description);
logErrorRecord(content.getId(), ErrorType.SITE_LIMIT, content.getUrl(), CONTENT_PART_IMAGE_LIST, description);
hasError = true;
} catch (PreparationInterruptedException ie) {
Timber.i(ie, "Preparation of %s interrupted", content.getTitle());
// not an error
} catch (EmptyResultException ere) {
Timber.i(ere, "No images have been found while parsing %s. Download aborted.", content.getTitle());
logErrorRecord(content.getId(), ErrorType.PARSING, content.getUrl(), CONTENT_PART_IMAGE_LIST, "No images have been found. Error = " + ere.getMessage());
hasError = true;
} catch (Exception e) {
Timber.w(e, "An exception has occurred while parsing %s. Download aborted.", content.getTitle());
logErrorRecord(content.getId(), ErrorType.PARSING, content.getUrl(), CONTENT_PART_IMAGE_LIST, e.getMessage());
hasError = true;
}
} else if (nbErrors > 0) {
// Other cases : Reset ERROR status of images to mark them as "to be downloaded" (in DB and in memory)
dao.updateImageContentStatus(content.getId(), StatusContent.ERROR, targetImageStatus);
} else {
if (downloadMode == Content.DownloadMode.STREAM)
dao.updateImageContentStatus(content.getId(), null, StatusContent.ONLINE);
}
// Get updated Content with the udpated ID and status of new images
content = dao.selectContent(content.getId());
if (null == content)
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
if (hasError) {
moveToErrors(content.getId());
EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.Type.EV_COMPLETE, 0, 0, 0, 0));
return new ImmutablePair<>(QueuingResult.CONTENT_FAILED, content);
}
// NB : No log of any sort because this is normal behaviour
if (downloadInterrupted.get())
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.PREPARE_FOLDER));
// Create destination folder for images to be downloaded
DocumentFile dir = ContentHelper.getOrCreateContentDownloadDir(getApplicationContext(), content);
// Folder creation failed
if (null == dir || !dir.exists()) {
String title = content.getTitle();
String absolutePath = (null == dir) ? "" : dir.getUri().toString();
String message = String.format("Directory could not be created: %s.", absolutePath);
Timber.w(message);
logErrorRecord(content.getId(), ErrorType.IO, content.getUrl(), "Destination folder", message);
notificationManager.notify(new DownloadWarningNotification(title, absolutePath));
// No sense in waiting for every image to be downloaded in error state (terrible waste of network resources)
// => Create all images, flag them as failed as well as the book
dao.updateImageContentStatus(content.getId(), targetImageStatus, StatusContent.ERROR);
completeDownload(content.getId(), content.getTitle(), 0, images.size(), 0);
return new ImmutablePair<>(QueuingResult.CONTENT_FAILED, content);
}
// Folder creation succeeds -> memorize its path
content.setStorageUri(dir.getUri().toString());
// Don't count the cover thumbnail in the number of pages
if (0 == content.getQtyPages())
content.setQtyPages(images.size() - 1);
content.setStatus(StatusContent.DOWNLOADING);
// Mark the cover for downloading when saving a streamed book
if (downloadMode == Content.DownloadMode.STREAM)
content.getCover().setStatus(StatusContent.SAVED);
dao.insertContent(content);
HentoidApp.trackDownloadEvent("Added");
Timber.i("Downloading '%s' [%s]", content.getTitle(), content.getId());
// Wait until the end of purge if the content is being purged (e.g. redownload from scratch)
boolean isBeingDeleted = content.isBeingDeleted();
if (isBeingDeleted)
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.WAIT_PURGE));
while (content.isBeingDeleted()) {
Timber.d("Waiting for purge to complete");
content = dao.selectContent(content.getId());
if (null == content)
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
Helper.pause(1000);
if (downloadInterrupted.get())
break;
}
if (isBeingDeleted && !downloadInterrupted.get())
Timber.d("Purge completed; resuming download");
// == DOWNLOAD PHASE ==
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.PREPARE_DOWNLOAD));
// Set up downloader constraints
if (content.getSite().getParallelDownloadCap() > 0 && (requestQueueManager.getDownloadThreadCap() > content.getSite().getParallelDownloadCap() || -1 == requestQueueManager.getDownloadThreadCap())) {
Timber.d("Setting parallel downloads count to %s", content.getSite().getParallelDownloadCap());
requestQueueManager.initUsingDownloadThreadCount(getApplicationContext(), content.getSite().getParallelDownloadCap(), true);
}
if (0 == content.getSite().getParallelDownloadCap() && requestQueueManager.getDownloadThreadCap() > -1) {
Timber.d("Resetting parallel downloads count to default");
requestQueueManager.initUsingDownloadThreadCount(getApplicationContext(), -1, true);
}
requestQueueManager.setNbRequestsPerSecond(content.getSite().getRequestsCapPerSecond());
// NB : No log of any sort because this is normal behaviour
if (downloadInterrupted.get())
return new ImmutablePair<>(QueuingResult.CONTENT_SKIPPED, null);
List<ImageFile> pagesToParse = new ArrayList<>();
List<ImageFile> ugoirasToDownload = new ArrayList<>();
// Just get the cover if we're in a streamed download
if (downloadMode == Content.DownloadMode.STREAM) {
Optional<ImageFile> coverOptional = Stream.of(images).filter(ImageFile::isCover).findFirst();
if (coverOptional.isPresent()) {
ImageFile cover = coverOptional.get();
enrichImageDownloadParams(cover, content);
requestQueueManager.queueRequest(buildImageDownloadRequest(cover, dir, content));
}
} else {
// Queue image download requests
for (ImageFile img : images) {
if (img.getStatus().equals(StatusContent.SAVED)) {
enrichImageDownloadParams(img, content);
// Set the 1st image of the list as a backup in case the cover URL is stale (might happen when restarting old downloads)
if (img.isCover() && images.size() > 1)
img.setBackupUrl(images.get(1).getUrl());
if (img.needsPageParsing())
pagesToParse.add(img);
else if (img.getDownloadParams().contains(ContentHelper.KEY_DL_PARAMS_UGOIRA_FRAMES))
ugoirasToDownload.add(img);
else
requestQueueManager.queueRequest(buildImageDownloadRequest(img, dir, content));
}
}
// Parse pages for images
if (!pagesToParse.isEmpty()) {
final Content contentFinal = content;
compositeDisposable.add(Observable.fromIterable(pagesToParse).observeOn(Schedulers.io()).subscribe(img -> parsePageforImage(img, dir, contentFinal), t -> {
// Nothing; just exit the Rx chain
}));
}
// Parse ugoiras for images
if (!ugoirasToDownload.isEmpty()) {
final Site siteFinal = content.getSite();
compositeDisposable.add(Observable.fromIterable(ugoirasToDownload).observeOn(Schedulers.io()).subscribe(img -> downloadAndUnzipUgoira(img, dir, siteFinal), t -> {
// Nothing; just exit the Rx chain
}));
}
}
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.SAVE_QUEUE));
if (ContentHelper.updateQueueJson(getApplicationContext(), dao))
Timber.i(context.getString(R.string.queue_json_saved));
else
Timber.w(context.getString(R.string.queue_json_failed));
EventBus.getDefault().post(DownloadEvent.fromPreparationStep(DownloadEvent.Step.START_DOWNLOAD));
return new ImmutablePair<>(QueuingResult.CONTENT_FOUND, content);
}
Aggregations