Examples with RescanableResourceWrapper org.commonjava.indy.subsys.prefetch.models.RescanableResourceWrapper used on opensource projects

Example 1 with RescanableResourceWrapper

use of org.commonjava.indy.subsys.prefetch.models.RescanableResourceWrapper in project indy by Commonjava.

the class PrefetchWorker method run.

@Override
public void run() {
    if (resources == null || resources.isEmpty()) {
        logger.trace("No resources for downloading");
        return;
    }
    logger.trace("Start downloading: {}", resources);
    final AtomicBoolean scheduled = new AtomicBoolean(false);
    for (Map.Entry<RemoteRepository, List<RescanableResourceWrapper>> entry : resources.entrySet()) {
        final RemoteRepository repo = entry.getKey();
        final List<RescanableResourceWrapper> res = entry.getValue();
        res.forEach(r -> {
            try {
                final String path = r.getResource().getPath();
                if (path == null || path.equals("") || path.endsWith("/") || path.endsWith(LISTING_HTML_FILE)) {
                    // If this is a rescan prefetch, we need to clear the listing cache and re-fetch from external
                    if (r.isRescan()) {
                        transfers.delete(new ConcreteResource(r.getResource().getLocation(), path, ".listing.txt"));
                    }
                    final List<RescanablePath> dirPaths = buildPaths(r.getResource(), r.isRescan());
                    logger.trace("{} is folder, will use it to schedule new Resources {}", r, dirPaths);
                    frontier.scheduleRepo(repo, dirPaths);
                    scheduled.set(true);
                } else {
                    // if repo has path masks, we need to check that first to only download path mask enabled artifacts.
                    if (PathMaskChecker.checkMask(repo, path)) {
                        // If this is a rescan prefetch, and artifact is metadata, we need to clear it and re-fetch from external
                        if (r.isRescan()) {
                            final SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(r.getResource());
                            if (spi != null && spi.isMetadata()) {
                                transfers.delete(r.getResource());
                            }
                        }
                        logger.trace("{} is file", r);
                        transfers.retrieve(r.getResource());
                    } else {
                        logger.trace("Path {} in repo {} not available for path mask {}", path, repo, repo.getPathMaskPatterns());
                    }
                }
            } catch (TransferException e) {
                logger.error("Download failed during prefetch because of transfer getting failed for {}, Reason: {}", r, e.getMessage());
            }
        });
    }
    if (scheduled.get()) {
        prefetchManager.triggerWorkers();
    }
}

Example 2 with RescanableResourceWrapper

use of org.commonjava.indy.subsys.prefetch.models.RescanableResourceWrapper in project indy by Commonjava.

the class PrefetchFrontier method remove.

public Map<RemoteRepository, List<RescanableResourceWrapper>> remove(final int size) {
    return lockAnd(t -> {
        Map<RemoteRepository, List<RescanableResourceWrapper>> resources = new HashMap<>(2);
        int removedSize = 0;
        final List<RemoteRepository> repoQueueCopy = new ArrayList<>(repoQueue);
        for (RemoteRepository repo : repoQueueCopy) {
            List<RescanablePath> paths = resourceCache.get(repo);
            if (paths != null) {
                List<RescanableResourceWrapper> res = new ArrayList<>(size);
                List<RescanablePath> pathsRemoved = new ArrayList<>(size);
                for (RescanablePath path : paths) {
                    res.add(new RescanableResourceWrapper(new StoreResource(LocationUtils.toLocation(repo), path.getPath()), path.isRescan()));
                    pathsRemoved.add(path);
                    if (++removedSize >= size) {
                        break;
                    }
                }
                resources.put(repo, res);
                paths.removeAll(pathsRemoved);
                if (paths.isEmpty()) {
                    resourceCache.remove(repo);
                    if (!repo.isPrefetchRescan()) {
                        repoQueue.remove(repo);
                        sortRepoQueue();
                    }
                    hasMore = !repoQueue.isEmpty() && !resourceCache.isEmpty();
                }
                if (removedSize >= size) {
                    return resources;
                }
            } else {
                if (!repo.isPrefetchRescan()) {
                    repoQueue.remove(repo);
                    sortRepoQueue();
                }
            }
        }
        return resources;
    });
}