Examples with StoreResource org.commonjava.indy.content.StoreResource used on opensource projects

Example 6 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class PrefetchFrontier method remove.

public Map<RemoteRepository, List<RescanableResourceWrapper>> remove(final int size) {
    return lockAnd(t -> {
        Map<RemoteRepository, List<RescanableResourceWrapper>> resources = new HashMap<>(2);
        int removedSize = 0;
        final List<RemoteRepository> repoQueueCopy = new ArrayList<>(repoQueue);
        for (RemoteRepository repo : repoQueueCopy) {
            List<RescanablePath> paths = resourceCache.get(repo);
            if (paths != null) {
                List<RescanableResourceWrapper> res = new ArrayList<>(size);
                List<RescanablePath> pathsRemoved = new ArrayList<>(size);
                for (RescanablePath path : paths) {
                    res.add(new RescanableResourceWrapper(new StoreResource(LocationUtils.toLocation(repo), path.getPath()), path.isRescan()));
                    pathsRemoved.add(path);
                    if (++removedSize >= size) {
                        break;
                    }
                }
                resources.put(repo, res);
                paths.removeAll(pathsRemoved);
                if (paths.isEmpty()) {
                    resourceCache.remove(repo);
                    if (!repo.isPrefetchRescan()) {
                        repoQueue.remove(repo);
                        sortRepoQueue();
                    }
                    hasMore = !repoQueue.isEmpty() && !resourceCache.isEmpty();
                }
                if (removedSize >= size) {
                    return resources;
                }
            } else {
                if (!repo.isPrefetchRescan()) {
                    repoQueue.remove(repo);
                    sortRepoQueue();
                }
            }
        }
        return resources;
    });
}

Example 7 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class HtmlContentListBuilder method buildContent.

@Override
public List<ConcreteResource> buildContent(final RemoteRepository repository, final boolean isRescan) {
    if (repository.getPrefetchPriority() <= 0) {
        logger.warn("The repository {} prefetch disabled, can not use html content listing", repository.getName());
        return Collections.emptyList();
    }
    final Set<String> pathMasks = repository.getPathMaskPatterns();
    boolean useDefault = true;
    if (pathMasks != null && !pathMasks.isEmpty()) {
        useDefault = pathMasks.stream().anyMatch(p -> PathMaskChecker.isRegexPattern(p));
    }
    if (useDefault) {
        final String rootPath = "/";
        final KeyedLocation loc = LocationUtils.toLocation(repository);
        final StoreResource res = new StoreResource(loc, rootPath);
        try {
            // If this is a rescan prefetch, we need to clear the listing cache and re-fetch from external
            if (isRescan) {
                transferManager.delete(new StoreResource(loc, "/.listing.txt"));
            }
            final ListingResult lr = transferManager.list(res);
            if (lr != null && lr.getListing() != null) {
                String[] files = lr.getListing();
                List<ConcreteResource> resources = new ArrayList<>(files.length);
                for (final String file : lr.getListing()) {
                    resources.add(new StoreResource(loc, rootPath, file));
                }
                return resources;
            }
        } catch (TransferException e) {
            logger.error(String.format("Can not get transfer for repository %s", repository), e);
        }
    } else {
        // if all path mask patterns are plaintext, we will use these as the download list directly.
        return pathMasks.stream().map(p -> new StoreResource(LocationUtils.toLocation(repository), p)).collect(Collectors.toList());
    }
    return Collections.emptyList();
}

Example 8 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class MavenMetadataGenerator method writeVersionMetadata.

private boolean writeVersionMetadata(final List<StoreResource> firstLevelFiles, final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    ArtifactPathInfo samplePomInfo = null;
    logger.debug("writeVersionMetadata, firstLevelFiles:{}, store:{}", firstLevelFiles, store.getKey());
    // first level will contain version directories...for each directory, we need to verify the presence of a .pom file before including
    // as a valid version
    final List<SingleVersion> versions = new ArrayList<>();
    nextTopResource: for (final StoreResource topResource : firstLevelFiles) {
        final String topPath = topResource.getPath();
        if (topPath.endsWith("/")) {
            final List<StoreResource> secondLevelListing = fileManager.listRaw(store, topPath);
            for (final StoreResource fileResource : secondLevelListing) {
                if (fileResource.getPath().endsWith(".pom")) {
                    ArtifactPathInfo filePomInfo = ArtifactPathInfo.parse(fileResource.getPath());
                    // check if the pom is valid for the path
                    if (filePomInfo != null) {
                        versions.add(VersionUtils.createSingleVersion(new File(topPath).getName()));
                        if (samplePomInfo == null) {
                            samplePomInfo = filePomInfo;
                        }
                        continue nextTopResource;
                    }
                }
            }
        }
    }
    if (versions.isEmpty()) {
        logger.debug("writeVersionMetadata, versions is empty, store:{}", store.getKey());
        return false;
    }
    logger.debug("writeVersionMetadata, versions: {}, store:{}", versions, store.getKey());
    Collections.sort(versions);
    final Transfer metadataFile = fileManager.getTransfer(store, path);
    OutputStream stream = null;
    try {
        final Document doc = xml.newDocumentBuilder().newDocument();
        final Map<String, String> coordMap = new HashMap<>();
        coordMap.put(ARTIFACT_ID, samplePomInfo == null ? null : samplePomInfo.getArtifactId());
        coordMap.put(GROUP_ID, samplePomInfo == null ? null : samplePomInfo.getGroupId());
        final String lastUpdated = generateUpdateTimestamp(getCurrentTimestamp());
        doc.appendChild(doc.createElementNS(doc.getNamespaceURI(), "metadata"));
        xml.createElement(doc.getDocumentElement(), null, coordMap);
        final Map<String, String> versioningMap = new HashMap<>();
        versioningMap.put(LAST_UPDATED, lastUpdated);
        final SingleVersion latest = versions.get(versions.size() - 1);
        versioningMap.put(LATEST, latest.renderStandard());
        SingleVersion release = null;
        for (int i = versions.size() - 1; i >= 0; i--) {
            final SingleVersion r = versions.get(i);
            if (r.isRelease()) {
                release = r;
                break;
            }
        }
        if (release != null) {
            versioningMap.put(RELEASE, release.renderStandard());
        }
        xml.createElement(doc, "versioning", versioningMap);
        final Element versionsElem = xml.createElement(doc, "versioning/versions", Collections.emptyMap());
        for (final SingleVersion version : versions) {
            final Element vElem = doc.createElement(VERSION);
            vElem.setTextContent(version.renderStandard());
            versionsElem.appendChild(vElem);
        }
        final String xmlStr = xml.toXML(doc, true);
        logger.debug("writeVersionMetadata, xmlStr: {}", xmlStr);
        stream = metadataFile.openOutputStream(TransferOperation.GENERATE, true, eventMetadata);
        stream.write(xmlStr.getBytes(UTF_8));
    } catch (final GalleyMavenXMLException e) {
        throw new IndyWorkflowException("Failed to generate maven metadata file: %s. Reason: %s", e, path, e.getMessage());
    } catch (final IOException e) {
        throw new IndyWorkflowException("Failed to write generated maven metadata file: %s. Reason: %s", e, metadataFile, e.getMessage());
    } finally {
        closeQuietly(stream);
    }
    logger.debug("writeVersionMetadata, DONE, store: {}", store.getKey());
    return true;
}

Example 9 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class MavenMetadataGenerator method writeSnapshotMetadata.

/**
 * First level will contain files that have the timestamp-buildNumber version suffix, e.g., 'o11yphant-metrics-api-1.0-20200805.065728-1.pom'
 * we need to parse each this info and add them to snapshot versions.
 */
private boolean writeSnapshotMetadata(final ArtifactPathInfo info, final List<StoreResource> firstLevelFiles, final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final Map<SnapshotPart, Set<ArtifactPathInfo>> infosBySnap = new HashMap<>();
    for (final StoreResource resource : firstLevelFiles) {
        final ArtifactPathInfo resInfo = ArtifactPathInfo.parse(resource.getPath());
        if (resInfo != null) {
            final SnapshotPart snap = resInfo.getSnapshotInfo();
            Set<ArtifactPathInfo> infos = infosBySnap.computeIfAbsent(snap, k -> new HashSet<>());
            infos.add(resInfo);
        }
    }
    if (infosBySnap.isEmpty()) {
        return false;
    }
    final List<SnapshotPart> snaps = new ArrayList<>(infosBySnap.keySet());
    Collections.sort(snaps);
    final Transfer metadataFile = fileManager.getTransfer(store, path);
    OutputStream stream = null;
    try {
        final Document doc = xml.newDocumentBuilder().newDocument();
        final Map<String, String> coordMap = new HashMap<>();
        coordMap.put(ARTIFACT_ID, info.getArtifactId());
        coordMap.put(GROUP_ID, info.getGroupId());
        coordMap.put(VERSION, info.getReleaseVersion() + LOCAL_SNAPSHOT_VERSION_PART);
        doc.appendChild(doc.createElementNS(doc.getNamespaceURI(), "metadata"));
        xml.createElement(doc.getDocumentElement(), null, coordMap);
        // the last one is the most recent
        SnapshotPart snap = snaps.get(snaps.size() - 1);
        Map<String, String> snapMap = new HashMap<>();
        if (snap.isLocalSnapshot()) {
            snapMap.put(LOCAL_COPY, Boolean.TRUE.toString());
        } else {
            snapMap.put(TIMESTAMP, SnapshotUtils.generateSnapshotTimestamp(snap.getTimestamp()));
            snapMap.put(BUILD_NUMBER, Integer.toString(snap.getBuildNumber()));
        }
        final Date currentTimestamp = getCurrentTimestamp();
        final String lastUpdated = getUpdateTimestamp(snap, currentTimestamp);
        xml.createElement(doc, "versioning", Collections.singletonMap(LAST_UPDATED, lastUpdated));
        xml.createElement(doc, "versioning/snapshot", snapMap);
        for (SnapshotPart snap1 : snaps) {
            final Set<ArtifactPathInfo> infos = infosBySnap.get(snap1);
            for (final ArtifactPathInfo pathInfo : infos) {
                snapMap = new HashMap<>();
                final TypeAndClassifier tc = new SimpleTypeAndClassifier(pathInfo.getType(), pathInfo.getClassifier());
                final TypeMapping mapping = typeMapper.lookup(tc);
                final String classifier = mapping == null ? pathInfo.getClassifier() : mapping.getClassifier();
                if (classifier != null && classifier.length() > 0) {
                    snapMap.put(CLASSIFIER, classifier);
                }
                snapMap.put(EXTENSION, mapping == null ? pathInfo.getType() : mapping.getExtension());
                snapMap.put(VALUE, pathInfo.getVersion());
                snapMap.put(UPDATED, getUpdateTimestamp(pathInfo.getSnapshotInfo(), currentTimestamp));
                xml.createElement(doc, "versioning/snapshotVersions/snapshotVersion", snapMap);
            }
        }
        final String xmlStr = xml.toXML(doc, true);
        stream = metadataFile.openOutputStream(TransferOperation.GENERATE, true, eventMetadata);
        stream.write(xmlStr.getBytes(UTF_8));
    } catch (final GalleyMavenXMLException e) {
        throw new IndyWorkflowException("Failed to generate maven metadata file: %s. Reason: %s", e, path, e.getMessage());
    } catch (final IOException e) {
        throw new IndyWorkflowException("Failed to write generated maven metadata file: %s. Reason: %s", e, metadataFile, e.getMessage());
    } finally {
        closeQuietly(stream);
    }
    return true;
}

Example 10 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class MavenMetadataGeneratorTest method setupVersionsStructureWith2Versions.

private StoreResource setupVersionsStructureWith2Versions() throws Exception {
    final RemoteRepository store = new RemoteRepository(MAVEN_PKG_KEY, "testrepo", "http://foo.bar");
    stores.storeArtifactStore(store, summary, false, true, new EventMetadata());
    final String path = "org/group/artifact";
    final KeyedLocation location = LocationUtils.toLocation(store);
    final StoreResource resource = new StoreResource(location, path);
    fixture.getTransport().registerListing(resource, new TestListing(new ListingResult(resource, new String[] { "1.0/", "1.1/" })));
    ConcreteResource versionDir = (resource.getChild("1.0/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.0.jar", "artifact-1.0.pom" })));
    versionDir = (resource.getChild("1.1/"));
    fixture.getTransport().registerListing(versionDir, new TestListing(new ListingResult(versionDir, new String[] { "artifact-1.1.jar", "artifact-1.1.pom" })));
    return resource;
}