Examples with RescanablePath org.commonjava.indy.subsys.prefetch.models.RescanablePath used on opensource projects

Example 1 with RescanablePath

use of org.commonjava.indy.subsys.prefetch.models.RescanablePath in project indy by Commonjava.

the class PrefetchManager method registerPrefetchStores.

public void registerPrefetchStores(@Observes final ArtifactStorePostUpdateEvent updateEvent) {
    if (config.isEnabled()) {
        logger.trace("Post update triggered for scheduling of prefetch: {}", updateEvent);
        final Collection<ArtifactStore> stores = updateEvent.getStores();
        boolean scheduled = false;
        for (ArtifactStore changedStore : stores) {
            if (changedStore.getType() == StoreType.remote) {
                ArtifactStore origStore = updateEvent.getOriginal(changedStore);
                final RemoteRepository changedRemote = (RemoteRepository) changedStore;
                final RemoteRepository origRemote = (RemoteRepository) origStore;
                final boolean remotePrefetchEnabled = (origRemote != null && !origRemote.getPrefetchPriority().equals(changedRemote.getPrefetchPriority())) && changedRemote.getPrefetchPriority() > 0;
                if (remotePrefetchEnabled) {
                    List<RescanablePath> paths = frontier.buildPaths(changedRemote, false);
                    logger.trace("Schedule resources: repo: {}, paths {}", changedRemote, paths);
                    frontier.scheduleRepo(changedRemote, paths);
                    scheduled = true;
                }
            }
        }
        if (scheduled) {
            triggerWorkers();
        }
    }
}

Example 2 with RescanablePath

use of org.commonjava.indy.subsys.prefetch.models.RescanablePath in project indy by Commonjava.

the class PrefetchWorker method run.

@Override
public void run() {
    if (resources == null || resources.isEmpty()) {
        logger.trace("No resources for downloading");
        return;
    }
    logger.trace("Start downloading: {}", resources);
    final AtomicBoolean scheduled = new AtomicBoolean(false);
    for (Map.Entry<RemoteRepository, List<RescanableResourceWrapper>> entry : resources.entrySet()) {
        final RemoteRepository repo = entry.getKey();
        final List<RescanableResourceWrapper> res = entry.getValue();
        res.forEach(r -> {
            try {
                final String path = r.getResource().getPath();
                if (path == null || path.equals("") || path.endsWith("/") || path.endsWith(LISTING_HTML_FILE)) {
                    // If this is a rescan prefetch, we need to clear the listing cache and re-fetch from external
                    if (r.isRescan()) {
                        transfers.delete(new ConcreteResource(r.getResource().getLocation(), path, ".listing.txt"));
                    }
                    final List<RescanablePath> dirPaths = buildPaths(r.getResource(), r.isRescan());
                    logger.trace("{} is folder, will use it to schedule new Resources {}", r, dirPaths);
                    frontier.scheduleRepo(repo, dirPaths);
                    scheduled.set(true);
                } else {
                    // if repo has path masks, we need to check that first to only download path mask enabled artifacts.
                    if (PathMaskChecker.checkMask(repo, path)) {
                        // If this is a rescan prefetch, and artifact is metadata, we need to clear it and re-fetch from external
                        if (r.isRescan()) {
                            final SpecialPathInfo spi = specialPathManager.getSpecialPathInfo(r.getResource());
                            if (spi != null && spi.isMetadata()) {
                                transfers.delete(r.getResource());
                            }
                        }
                        logger.trace("{} is file", r);
                        transfers.retrieve(r.getResource());
                    } else {
                        logger.trace("Path {} in repo {} not available for path mask {}", path, repo, repo.getPathMaskPatterns());
                    }
                }
            } catch (TransferException e) {
                logger.error("Download failed during prefetch because of transfer getting failed for {}, Reason: {}", r, e.getMessage());
            }
        });
    }
    if (scheduled.get()) {
        prefetchManager.triggerWorkers();
    }
}

Example 3 with RescanablePath

use of org.commonjava.indy.subsys.prefetch.models.RescanablePath in project indy by Commonjava.

the class PrefetchWorker method buildPaths.

private List<RescanablePath> buildPaths(final ConcreteResource resource, final boolean isRescan) {
    try {
        ListingResult lr = transfers.list(resource);
        if (lr != null && lr.getListing() != null) {
            String[] files = lr.getListing();
            List<RescanablePath> paths = new ArrayList<>(files.length);
            for (final String file : files) {
                paths.add(new RescanablePath(PathUtils.normalize(resource.getPath(), file), isRescan));
            }
            return paths;
        } else {
            logger.trace("No content found for {}", resource);
        }
    } catch (TransferException e) {
        logger.error("List content failed during prefetch because of transfer getting failed for {}, Reason: {}", resource, e.getMessage());
    }
    return Collections.emptyList();
}

Example 4 with RescanablePath

use of org.commonjava.indy.subsys.prefetch.models.RescanablePath in project indy by Commonjava.

the class PrefetchFrontier method get.

public Map<RemoteRepository, List<ConcreteResource>> get(final int size) {
    return lockAnd(t -> {
        Map<RemoteRepository, List<ConcreteResource>> resources = new HashMap<>(2);
        int removedSize = 0;
        for (RemoteRepository repo : repoQueue) {
            List<RescanablePath> paths = resourceCache.get(repo);
            if (paths != null && !paths.isEmpty()) {
                List<ConcreteResource> res = new ArrayList<>(size);
                for (RescanablePath path : paths) {
                    res.add(new StoreResource(LocationUtils.toLocation(repo), path.getPath()));
                }
                resources.put(repo, res);
                if (removedSize >= size) {
                    return resources;
                }
            }
        }
        return resources;
    });
}

Example 5 with RescanablePath

use of org.commonjava.indy.subsys.prefetch.models.RescanablePath in project indy by Commonjava.

the class PrefetchFrontier method remove.

public Map<RemoteRepository, List<RescanableResourceWrapper>> remove(final int size) {
    return lockAnd(t -> {
        Map<RemoteRepository, List<RescanableResourceWrapper>> resources = new HashMap<>(2);
        int removedSize = 0;
        final List<RemoteRepository> repoQueueCopy = new ArrayList<>(repoQueue);
        for (RemoteRepository repo : repoQueueCopy) {
            List<RescanablePath> paths = resourceCache.get(repo);
            if (paths != null) {
                List<RescanableResourceWrapper> res = new ArrayList<>(size);
                List<RescanablePath> pathsRemoved = new ArrayList<>(size);
                for (RescanablePath path : paths) {
                    res.add(new RescanableResourceWrapper(new StoreResource(LocationUtils.toLocation(repo), path.getPath()), path.isRescan()));
                    pathsRemoved.add(path);
                    if (++removedSize >= size) {
                        break;
                    }
                }
                resources.put(repo, res);
                paths.removeAll(pathsRemoved);
                if (paths.isEmpty()) {
                    resourceCache.remove(repo);
                    if (!repo.isPrefetchRescan()) {
                        repoQueue.remove(repo);
                        sortRepoQueue();
                    }
                    hasMore = !repoQueue.isEmpty() && !resourceCache.isEmpty();
                }
                if (removedSize >= size) {
                    return resources;
                }
            } else {
                if (!repo.isPrefetchRescan()) {
                    repoQueue.remove(repo);
                    sortRepoQueue();
                }
            }
        }
        return resources;
    });
}