Examples with KeyedLocation org.commonjava.indy.model.galley.KeyedLocation used on opensource projects

Example 1 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class IndexingContentManagerDecorator method retrieve.

@Override
public Transfer retrieve(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.trace("Looking for indexed path: {} in: {}", path, store.getKey());
    Transfer transfer = getIndexedTransfer(store.getKey(), null, path, TransferOperation.DOWNLOAD);
    if (transfer != null) {
        logger.debug("Found indexed transfer: {}. Returning.", transfer);
        return transfer;
    }
    StoreType type = store.getKey().getType();
    if (StoreType.group == type) {
        ConcreteResource resource = new ConcreteResource(LocationUtils.toLocation(store), path);
        if (nfc.isMissing(resource)) {
            logger.debug("{} is marked as missing. Returning null.", resource);
            return null;
        }
        logger.debug("No group index hits. Devolving to member store indexes.");
        KeyedLocation location = LocationUtils.toLocation(store);
        SpecialPathInfo specialPathInfo = specialPathManager.getSpecialPathInfo(location, path, store.getPackageType());
        if (specialPathInfo == null || !specialPathInfo.isMergable()) {
            transfer = getIndexedMemberTransfer((Group) store, path, TransferOperation.DOWNLOAD, (member) -> {
                try {
                    return delegate.retrieve(member, path);
                } catch (IndyWorkflowException e) {
                    logger.error(String.format("Failed to retrieve() for member path: %s:%s. Reason: %s", member.getKey(), path, e.getMessage()), e);
                }
                return null;
            });
            if (transfer != null) {
                nfc.clearMissing(resource);
                return transfer;
            }
            logger.debug("No index hits. Delegating to main content manager for: {} in: {}", path, store);
        } else {
            logger.debug("Merged content. Delegating to main content manager for: {} in: {}", path, store);
            transfer = delegate.retrieve(store, path, eventMetadata);
            if (transfer == null) {
                nfc.addMissing(resource);
            }
            return transfer;
        }
    }
    transfer = delegate.retrieve(store, path, eventMetadata);
    if (transfer != null) {
        logger.debug("Got transfer from delegate: {} (will index)", transfer);
        indexManager.indexTransferIn(transfer, store.getKey());
    }
    logger.debug("Returning transfer: {}", transfer);
    return transfer;
}

Example 2 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class MavenMetadataGeneratorTest method setupVersionsStructureWith2Versions.

private StoreResource setupVersionsStructureWith2Versions() throws Exception {
    final RemoteRepository store = new RemoteRepository(MAVEN_PKG_KEY, "testrepo", "http://foo.bar");
    stores.storeArtifactStore(store, summary, false, true, new EventMetadata());
    final String path = "org/group/artifact";
    final KeyedLocation location = LocationUtils.toLocation(store);
    final StoreResource resource = new StoreResource(location, path);
    fixture.getTransport().registerListing(resource, new TestListing(new ListingResult(resource, new String[] { "1.0/", "1.1/" })));
    ConcreteResource versionDir = (resource.getChild("1.0/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.0.jar", "artifact-1.0.pom" })));
    versionDir = (resource.getChild("1.1/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.1.jar", "artifact-1.1.pom" })));
    return resource;
}

Example 3 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class ContentController method renderListing.

public String renderListing(final String acceptHeader, final StoreKey key, final String requestPath, final String serviceUrl, final UriFormatter uriFormatter) throws IndyWorkflowException {
    String path = requestPath;
    if (path.endsWith(LISTING_HTML_FILE)) {
        path = normalize(parentPath(path));
    }
    final List<StoreResource> listed = getListing(key, path);
    if (ApplicationContent.application_json.equals(acceptHeader)) {
        final DirectoryListingDTO dto = new DirectoryListingDTO(StoreResource.convertToEntries(listed));
        try {
            return mapper.writeValueAsString(dto);
        } catch (final JsonProcessingException e) {
            throw new IndyWorkflowException("Failed to render listing to JSON: %s. Reason: %s", e, dto, e.getMessage());
        }
    }
    final Map<String, Set<String>> listingUrls = new TreeMap<>();
    final String storeUrl = uriFormatter.formatAbsolutePathTo(serviceUrl, key.getType().singularEndpointName(), key.getName());
    if (listed != null) {
        // second pass, process the remainder.
        for (int pass = 0; pass < 2; pass++) {
            for (final ConcreteResource res : listed) {
                String p = res.getPath();
                if (pass == 0 && !p.endsWith("/")) {
                    continue;
                } else if (pass == 1) {
                    if (!p.endsWith("/")) {
                        final String dirpath = p + "/";
                        if (listingUrls.containsKey(normalize(storeUrl, dirpath))) {
                            p = dirpath;
                        }
                    } else {
                        continue;
                    }
                }
                final String localUrl = normalize(storeUrl, p);
                Set<String> sources = listingUrls.get(localUrl);
                if (sources == null) {
                    sources = new HashSet<>();
                    listingUrls.put(localUrl, sources);
                }
                sources.add(normalize(res.getLocationUri(), res.getPath()));
            }
        }
    }
    final List<String> sources = new ArrayList<>();
    if (listed != null) {
        for (final ConcreteResource res : listed) {
            // KeyedLocation is all we use in Indy.
            logger.debug("Formatting sources URL for: {}", res);
            final KeyedLocation kl = (KeyedLocation) res.getLocation();
            final String uri = uriFormatter.formatAbsolutePathTo(serviceUrl, kl.getKey().getType().singularEndpointName(), kl.getKey().getName());
            if (!sources.contains(uri)) {
                logger.debug("adding source URI: '{}'", uri);
                sources.add(uri);
            }
        }
    }
    Collections.sort(sources);
    String parentPath = normalize(parentPath(path));
    if (!parentPath.endsWith("/")) {
        parentPath += "/";
    }
    final String parentUrl;
    if (parentPath.equals(path)) {
        parentPath = null;
        parentUrl = null;
    } else {
        parentUrl = uriFormatter.formatAbsolutePathTo(serviceUrl, key.getType().singularEndpointName(), key.getName(), parentPath);
    }
    final Map<String, Object> params = new HashMap<>();
    params.put("items", listingUrls);
    params.put("parentUrl", parentUrl);
    params.put("parentPath", parentPath);
    params.put("path", path);
    params.put("storeKey", key);
    params.put("storeUrl", storeUrl);
    params.put("baseUrl", serviceUrl);
    params.put("sources", sources);
    // render...
    try {
        return templates.render(acceptHeader, "directory-listing", params);
    } catch (final IndyGroovyException e) {
        throw new IndyWorkflowException(e.getMessage(), e);
    }
}

Example 4 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class NfcController method getMissing.

public NotFoundCacheDTO getMissing(final StoreKey key) throws IndyWorkflowException {
    final NotFoundCacheDTO dto = new NotFoundCacheDTO();
    if (key.getType() == group) {
        List<ArtifactStore> stores;
        try {
            stores = storeManager.query().packageType(key.getPackageType()).getOrderedConcreteStoresInGroup(key.getName());
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Failed to retrieve concrete constituent ArtifactStores for: %s.", e, key);
        }
        final List<? extends KeyedLocation> locations = toLocations(stores);
        for (final KeyedLocation location : locations) {
            final Set<String> missing = cache.getMissing(location);
            if (missing != null && !missing.isEmpty()) {
                final List<String> paths = new ArrayList<String>(missing);
                Collections.sort(paths);
                dto.addSection(location.getKey(), paths);
            }
        }
    } else {
        ArtifactStore store;
        try {
            store = storeManager.getArtifactStore(key);
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Failed to retrieve ArtifactStore: %s.", e, key);
        }
        if (store != null) {
            final Set<String> missing = cache.getMissing(toLocation(store));
            final List<String> paths = new ArrayList<String>(missing);
            Collections.sort(paths);
            dto.addSection(key, paths);
        }
    }
    return dto;
}

Example 5 with KeyedLocation

use of org.commonjava.indy.model.galley.KeyedLocation in project indy by Commonjava.

the class DefaultDownloadManager method list.

@Override
public List<StoreResource> list(final List<? extends ArtifactStore> stores, final String path) throws IndyWorkflowException {
    final String dir = PathUtils.dirname(path);
    final List<StoreResource> result = new ArrayList<>();
    try {
        final List<ListingResult> results = transfers.listAll(locationExpander.expand(new VirtualResource(LocationUtils.toLocations(stores), path)));
        for (final ListingResult lr : results) {
            if (lr != null && lr.getListing() != null) {
                for (final String file : lr.getListing()) {
                    result.add(new StoreResource((KeyedLocation) lr.getLocation(), dir, file));
                }
            }
        }
    } catch (final BadGatewayException e) {
        Location location = e.getLocation();
        KeyedLocation kl = (KeyedLocation) location;
        fileEventManager.fire(new IndyStoreErrorEvent(kl.getKey(), e));
        logger.warn("Bad gateway: " + e.getMessage(), e);
    } catch (final TransferTimeoutException e) {
        Location location = e.getLocation();
        KeyedLocation kl = (KeyedLocation) location;
        fileEventManager.fire(new IndyStoreErrorEvent(kl.getKey(), e));
        logger.warn("Timeout: " + e.getMessage(), e);
    } catch (final TransferLocationException e) {
        Location location = e.getLocation();
        KeyedLocation kl = (KeyedLocation) location;
        fileEventManager.fire(new IndyStoreErrorEvent(kl.getKey(), e));
        logger.warn("Location Error: " + e.getMessage(), e);
    } catch (final TransferException e) {
        logger.error(e.getMessage(), e);
        throw new IndyWorkflowException("Failed to list ALL paths: {} from: {}. Reason: {}", e, path, stores, e.getMessage());
    }
    return dedupeListing(result);
}