Examples with ListingResult org.commonjava.maven.galley.model.ListingResult used on opensource projects

Example 6 with ListingResult

use of org.commonjava.maven.galley.model.ListingResult in project galley by Commonjava.

the class HttpListing method call.

@Override
public ListingResult call() {
    request = new HttpGet(url);
    // return null if something goes wrong, after setting the error.
    // What we should be doing here is trying to retrieve the html directory
    // listing, then parse out the filenames from that...
    // 
    // They'll be links, so that's something to key in on.
    // 
    // I'm wondering about this:
    // http://jsoup.org/cookbook/extracting-data/selector-syntax
    // the dependency is: org.jsoup:jsoup:1.7.2
    ListingResult result = null;
    InputStream in = null;
    try {
        if (executeHttp()) {
            in = response.getEntity().getContent();
            String listing = IOUtils.toString(in);
            Logger logger = LoggerFactory.getLogger(getClass());
            logger.debug("Got raw listing content:\n\n{}\n\n", listing);
            final ArrayList<String> al = new ArrayList<>();
            // TODO: Charset!!
            Document doc = Jsoup.parse(listing, url);
            if (doc != null) {
                for (final Element link : doc.select("a")) {
                    String linkText = link.text();
                    String linkHref = link.attr("href");
                    URL url = new URL(this.url);
                    boolean sameServer = isSameServer(url, linkHref);
                    boolean subpath = isSubpath(url, linkHref);
                    if ((sameServer && subpath) && (linkHref.endsWith(linkText) || linkHref.endsWith(linkText + '/')) && !EXCLUDES.contains(linkText)) {
                        al.add(linkText);
                    }
                }
                result = new ListingResult(resource, al.toArray(new String[al.size()]));
            }
        }
    } catch (final TransferException e) {
        this.error = e;
    } catch (final IOException e) {
        this.error = new TransferException("Failed to construct directory listing for: {}. Reason: {}", e, url, e.getMessage());
    } finally {
        closeQuietly(in);
        cleanup();
    }
    return error == null ? result : null;
}

Example 7 with ListingResult

use of org.commonjava.maven.galley.model.ListingResult in project indy by Commonjava.

the class MavenMetadataGeneratorTest method setupVersionsStructureWith2Versions.

private StoreResource setupVersionsStructureWith2Versions() throws Exception {
    final RemoteRepository store = new RemoteRepository(MAVEN_PKG_KEY, "testrepo", "http://foo.bar");
    stores.storeArtifactStore(store, summary, false, true, new EventMetadata());
    final String path = "org/group/artifact";
    final KeyedLocation location = LocationUtils.toLocation(store);
    final StoreResource resource = new StoreResource(location, path);
    fixture.getTransport().registerListing(resource, new TestListing(new ListingResult(resource, new String[] { "1.0/", "1.1/" })));
    ConcreteResource versionDir = (resource.getChild("1.0/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.0.jar", "artifact-1.0.pom" })));
    versionDir = (resource.getChild("1.1/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.1.jar", "artifact-1.1.pom" })));
    return resource;
}

Example 8 with ListingResult

use of org.commonjava.maven.galley.model.ListingResult in project indy by Commonjava.

the class HtmlContentListBuilder method buildContent.

@Override
public List<ConcreteResource> buildContent(final RemoteRepository repository, final boolean isRescan) {
    if (repository.getPrefetchPriority() <= 0) {
        logger.warn("The repository {} prefetch disabled, can not use html content listing", repository.getName());
        return Collections.emptyList();
    }
    final Set<String> pathMasks = repository.getPathMaskPatterns();
    boolean useDefault = true;
    if (pathMasks != null && !pathMasks.isEmpty()) {
        useDefault = pathMasks.stream().anyMatch(p -> PathMaskChecker.isRegexPattern(p));
    }
    if (useDefault) {
        final String rootPath = "/";
        final KeyedLocation loc = LocationUtils.toLocation(repository);
        final StoreResource res = new StoreResource(loc, rootPath);
        try {
            // If this is a rescan prefetch, we need to clear the listing cache and re-fetch from external
            if (isRescan) {
                transferManager.delete(new StoreResource(loc, "/.listing.txt"));
            }
            final ListingResult lr = transferManager.list(res);
            if (lr != null && lr.getListing() != null) {
                String[] files = lr.getListing();
                List<ConcreteResource> resources = new ArrayList<>(files.length);
                for (final String file : lr.getListing()) {
                    resources.add(new StoreResource(loc, rootPath, file));
                }
                return resources;
            }
        } catch (TransferException e) {
            logger.error(String.format("Can not get transfer for repository %s", repository), e);
        }
    } else {
        // if all path mask patterns are plaintext, we will use these as the download list directly.
        return pathMasks.stream().map(p -> new StoreResource(LocationUtils.toLocation(repository), p)).collect(Collectors.toList());
    }
    return Collections.emptyList();
}

Example 9 with ListingResult

use of org.commonjava.maven.galley.model.ListingResult in project indy by Commonjava.

the class PrefetchWorker method buildPaths.

private List<RescanablePath> buildPaths(final ConcreteResource resource, final boolean isRescan) {
    try {
        ListingResult lr = transfers.list(resource);
        if (lr != null && lr.getListing() != null) {
            String[] files = lr.getListing();
            List<RescanablePath> paths = new ArrayList<>(files.length);
            for (final String file : files) {
                paths.add(new RescanablePath(PathUtils.normalize(resource.getPath(), file), isRescan));
            }
            return paths;
        } else {
            logger.trace("No content found for {}", resource);
        }
    } catch (TransferException e) {
        logger.error("List content failed during prefetch because of transfer getting failed for {}, Reason: {}", resource, e.getMessage());
    }
    return Collections.emptyList();
}

Example 10 with ListingResult

use of org.commonjava.maven.galley.model.ListingResult in project indy by Commonjava.

the class DefaultDownloadManager method list.

@Override
@Measure
public List<StoreResource> list(final List<? extends ArtifactStore> stores, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final String dir = PathUtils.dirname(path);
    final List<StoreResource> result = new ArrayList<>();
    try {
        final List<ListingResult> results = transfers.listAll(locationExpander.expand(new VirtualResource(LocationUtils.toLocations(stores), path)), eventMetadata);
        for (final ListingResult lr : results) {
            if (lr != null && lr.getListing() != null) {
                for (final String file : lr.getListing()) {
                    result.add(new StoreResource((KeyedLocation) lr.getLocation(), dir, file));
                }
            }
        }
    } catch (final TransferLocationException e) {
        fireIndyStoreErrorEvent(e);
        logger.warn("Location Error: " + e.getMessage(), e);
        throw new IndyWorkflowException("Failed to list ALL paths: {} from: {}. Reason: {}", e, path, stores, e.getMessage());
    } catch (final TransferException e) {
        logger.error(e.getMessage(), e);
        throw new IndyWorkflowException("Failed to list ALL paths: {} from: {}. Reason: {}", e, path, stores, e.getMessage());
    }
    return dedupeListing(result);
}