Search in sources :

Example 21 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class MavenMetadataGenerator method generateDirectoryContent.

@Override
public List<StoreResource> generateDirectoryContent(final ArtifactStore store, final String path, final List<StoreResource> existing, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final StoreResource mdResource = new StoreResource(LocationUtils.toLocation(store), Paths.get(path, MavenMetadataMerger.METADATA_NAME).toString());
    if (existing.contains(mdResource)) {
        return null;
    }
    int pathElementsCount = StringUtils.strip(path, "/").split("/").length;
    // if there is a possibility we are listing an artifactId
    if (pathElementsCount >= 2) {
        // regardless, we will need this first level of listings. What we do with it will depend on the logic below...
        final List<StoreResource> firstLevelFiles = fileManager.listRaw(store, path, eventMetadata);
        ArtifactPathInfo samplePomInfo = null;
        for (final StoreResource topResource : firstLevelFiles) {
            final String topPath = topResource.getPath();
            if (topPath.endsWith(".pom")) {
                samplePomInfo = ArtifactPathInfo.parse(topPath);
                break;
            }
        }
        // if this dir does not contain a pom check if a subdir contain a pom
        if (samplePomInfo == null) {
            List<String> firstLevelDirs = firstLevelFiles.stream().map(ConcreteResource::getPath).filter((subpath) -> subpath.endsWith("/")).collect(Collectors.toList());
            final Map<String, List<StoreResource>> secondLevelMap = fileManager.listRaw(store, firstLevelDirs, eventMetadata);
            nextTopResource: for (final String topPath : firstLevelDirs) {
                final List<StoreResource> secondLevelListing = secondLevelMap.get(topPath);
                for (final StoreResource fileResource : secondLevelListing) {
                    if (fileResource.getPath().endsWith(".pom")) {
                        samplePomInfo = ArtifactPathInfo.parse(fileResource.getPath());
                        break nextTopResource;
                    }
                }
            }
        }
        // We won't worry about this for now.
        if (samplePomInfo != null) {
            final List<StoreResource> result = new ArrayList<>();
            result.add(mdResource);
            result.add(new StoreResource(LocationUtils.toLocation(store), Paths.get(path, MavenMetadataMerger.METADATA_MD5_NAME).toString()));
            result.add(new StoreResource(LocationUtils.toLocation(store), Paths.get(path, MavenMetadataMerger.METADATA_SHA_NAME).toString()));
            return result;
        }
    }
    return null;
}
Also used : HTTP_METADATA_EXT(org.commonjava.maven.galley.io.SpecialPathConstants.HTTP_METADATA_EXT) Date(java.util.Date) BiFunction(java.util.function.BiFunction) MergedContentAction(org.commonjava.indy.content.MergedContentAction) PathUtils.normalize(org.commonjava.maven.galley.util.PathUtils.normalize) StringUtils(org.apache.commons.lang3.StringUtils) WeftExecutorService(org.commonjava.cdi.util.weft.WeftExecutorService) GROUP_METADATA_GENERATED(org.commonjava.indy.core.content.group.GroupMergeHelper.GROUP_METADATA_GENERATED) WeftManaged(org.commonjava.cdi.util.weft.WeftManaged) TransferOperation(org.commonjava.maven.galley.model.TransferOperation) Document(org.w3c.dom.Document) TraceManager.addFieldToActiveSpan(org.commonjava.o11yphant.trace.TraceManager.addFieldToActiveSpan) Map(java.util.Map) StoreKey(org.commonjava.indy.model.core.StoreKey) Instance(javax.enterprise.inject.Instance) GROUP_METADATA_EXISTS(org.commonjava.indy.core.content.group.GroupMergeHelper.GROUP_METADATA_EXISTS) ArtifactPathInfo(org.commonjava.atlas.maven.ident.util.ArtifactPathInfo) GroupMergeHelper(org.commonjava.indy.core.content.group.GroupMergeHelper) MavenMetadataMerger(org.commonjava.indy.pkg.maven.content.group.MavenMetadataMerger) ConcreteResource(org.commonjava.maven.galley.model.ConcreteResource) StoreType(org.commonjava.indy.model.core.StoreType) Set(java.util.Set) Collectors(java.util.stream.Collectors) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) Metadata(org.apache.maven.artifact.repository.metadata.Metadata) LOCAL_SNAPSHOT_VERSION_PART(org.commonjava.atlas.maven.ident.util.SnapshotUtils.LOCAL_SNAPSHOT_VERSION_PART) PostConstruct(javax.annotation.PostConstruct) XMLInfrastructure(org.commonjava.maven.galley.maven.parse.XMLInfrastructure) DirectContentAccess(org.commonjava.indy.content.DirectContentAccess) PoolUtils.detectOverloadVoid(org.commonjava.indy.core.ctl.PoolUtils.detectOverloadVoid) Plugin(org.apache.maven.artifact.repository.metadata.Plugin) ByteArrayOutputStream(java.io.ByteArrayOutputStream) MetadataXpp3Writer(org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer) Locker(org.commonjava.cdi.util.weft.Locker) MavenMetadataProvider(org.commonjava.indy.pkg.maven.content.group.MavenMetadataProvider) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) AtomicReference(java.util.concurrent.atomic.AtomicReference) VersionUtils(org.commonjava.atlas.maven.ident.util.VersionUtils) Group(org.commonjava.indy.model.core.Group) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) SnapshotVersion(org.apache.maven.artifact.repository.metadata.SnapshotVersion) Transfer(org.commonjava.maven.galley.model.Transfer) Measure(org.commonjava.o11yphant.metrics.annotation.Measure) ExecutorConfig(org.commonjava.cdi.util.weft.ExecutorConfig) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) TypeAndClassifier(org.commonjava.atlas.maven.ident.ref.TypeAndClassifier) GalleyMavenXMLException(org.commonjava.maven.galley.maven.parse.GalleyMavenXMLException) PathUtils.parentPath(org.commonjava.maven.galley.util.PathUtils.parentPath) DrainingExecutorCompletionService(org.commonjava.cdi.util.weft.DrainingExecutorCompletionService) SnapshotPart(org.commonjava.atlas.maven.ident.version.part.SnapshotPart) MetadataXpp3Reader(org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader) OutputStream(java.io.OutputStream) Versioning(org.apache.maven.artifact.repository.metadata.Versioning) LocationUtils(org.commonjava.indy.util.LocationUtils) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) TraceManager(org.commonjava.o11yphant.trace.TraceManager) IOUtils.closeQuietly(org.apache.commons.io.IOUtils.closeQuietly) IOException(java.io.IOException) SnapshotUtils.generateUpdateTimestamp(org.commonjava.atlas.maven.ident.util.SnapshotUtils.generateUpdateTimestamp) SimpleTypeAndClassifier(org.commonjava.atlas.maven.ident.ref.SimpleTypeAndClassifier) SnapshotUtils(org.commonjava.atlas.maven.ident.util.SnapshotUtils) NotFoundCache(org.commonjava.maven.galley.spi.nfc.NotFoundCache) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) Element(org.w3c.dom.Element) StringReader(java.io.StringReader) Paths(java.nio.file.Paths) TypeMapper(org.commonjava.maven.galley.maven.spi.type.TypeMapper) StoreResource(org.commonjava.indy.content.StoreResource) SnapshotUtils.getCurrentTimestamp(org.commonjava.atlas.maven.ident.util.SnapshotUtils.getCurrentTimestamp) TypeMapping(org.commonjava.maven.galley.model.TypeMapping) SingleVersion(org.commonjava.atlas.maven.ident.version.SingleVersion) AbstractMergedContentGenerator(org.commonjava.indy.core.content.AbstractMergedContentGenerator) Collections(java.util.Collections) StoreDataManager(org.commonjava.indy.data.StoreDataManager) InputStream(java.io.InputStream) StoreResource(org.commonjava.indy.content.StoreResource) ArtifactPathInfo(org.commonjava.atlas.maven.ident.util.ArtifactPathInfo) ConcreteResource(org.commonjava.maven.galley.model.ConcreteResource) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList)

Example 22 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class MavenMetadataGenerator method generateFileContent.

@Override
@Measure
public Transfer generateFileContent(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    // metadata merging is something else...don't handle it here.
    if (StoreType.group == store.getKey().getType()) {
        return null;
    }
    if (!canProcess(path)) {
        return null;
    }
    boolean generated;
    // TODO: Generation of plugin metadata files (groupId-level) is harder, and requires cracking open the jar file
    // This is because that's the only place the plugin prefix can be reliably retrieved from.
    // regardless, we will need this first level of listings. What we do with it will depend on the logic below...
    final String parentPath = Paths.get(path).getParent().toString();
    List<StoreResource> firstLevel;
    try {
        firstLevel = fileManager.listRaw(store, parentPath);
    } catch (final IndyWorkflowException e) {
        logger.error(String.format("SKIP: Failed to generate maven-metadata.xml from listing of directory contents for: %s under path: %s", store, parentPath), e);
        return null;
    }
    String toGenPath = path;
    if (!path.endsWith(MavenMetadataMerger.METADATA_NAME)) {
        toGenPath = normalize(normalize(parentPath(toGenPath)), MavenMetadataMerger.METADATA_NAME);
    }
    ArtifactPathInfo snapshotPomInfo = null;
    if (parentPath.endsWith(LOCAL_SNAPSHOT_VERSION_PART)) {
        // If we're in a version directory, first-level listing should include a .pom file
        for (final StoreResource resource : firstLevel) {
            if (resource.getPath().endsWith(".pom")) {
                snapshotPomInfo = ArtifactPathInfo.parse(resource.getPath());
                break;
            }
        }
    }
    if (snapshotPomInfo != null) {
        logger.debug("Generating maven-metadata.xml for snapshots, store: {}", store.getKey());
        generated = writeSnapshotMetadata(snapshotPomInfo, firstLevel, store, toGenPath, eventMetadata);
    } else {
        logger.debug("Generating maven-metadata.xml for releases, store: {}", store.getKey());
        generated = writeVersionMetadata(firstLevel, store, toGenPath, eventMetadata);
    }
    logger.debug("[Result] Generating maven-metadata.xml for store: {}, result: {}", store.getKey(), generated);
    return generated ? fileManager.getTransfer(store, path) : null;
}
Also used : StoreResource(org.commonjava.indy.content.StoreResource) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) ArtifactPathInfo(org.commonjava.atlas.maven.ident.util.ArtifactPathInfo) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 23 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class DefaultContentManager method list.

@Override
@Measure
public List<StoreResource> list(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final EventMetadata metadata = setAllowRemoteDownloading(eventMetadata);
    List<StoreResource> listed;
    if (group == store.getKey().getType()) {
        List<ArtifactStore> members = getOrderedConcreteStoresAndFilter((Group) store, path);
        listed = new ArrayList<>();
        contentGeneratorManager.generateGroupDirectoryContentAnd((Group) store, members, path, eventMetadata, listed::addAll);
        for (final ArtifactStore member : members) {
            List<StoreResource> storeListing = null;
            try {
                storeListing = list(member, path, metadata);
            } catch (IndyWorkflowException e) {
                e.filterLocationErrors();
            }
            if (storeListing != null) {
                listed.addAll(storeListing);
            }
        }
    } else {
        listed = downloadManager.list(store, path, metadata);
        contentGeneratorManager.generateDirectoryContentAnd(store, path, listed, metadata, listed::addAll);
    }
    return dedupeListing(listed);
}
Also used : StoreResource(org.commonjava.indy.content.StoreResource) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 24 with StoreResource

use of org.commonjava.indy.content.StoreResource in project indy by Commonjava.

the class DefaultDownloadManager method list.

@Override
@Measure
public List<StoreResource> list(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    final List<StoreResource> result = new ArrayList<>();
    if (store.getKey().getType() == StoreType.group) {
        try {
            final List<ListingResult> results = transfers.listAll(locationExpander.expand(new VirtualResource(LocationUtils.toLocations(store), path)), eventMetadata);
            for (final ListingResult lr : results) {
                if (lr != null && lr.getListing() != null) {
                    for (final String file : lr.getListing()) {
                        result.add(new StoreResource((KeyedLocation) lr.getLocation(), path, file));
                    }
                }
            }
        } catch (final TransferLocationException e) {
            fireIndyStoreErrorEvent(e);
            logger.warn("Location Error: " + e.getMessage(), e);
            throw new IndyWorkflowException("Failed to list ALL paths: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
        } catch (final TransferException e) {
            logger.error(e.getMessage(), e);
            throw new IndyWorkflowException("Failed to list ALL paths: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
        }
    } else {
        if (!PathMaskChecker.checkListingMask(store, path)) {
            // if list not permitted for the path, return empty list
            return result;
        }
        final KeyedLocation loc = LocationUtils.toLocation(store);
        final StoreResource res = new StoreResource(loc, path);
        if (store instanceof RemoteRepository) {
            try {
                final ListingResult lr = transfers.list(res, eventMetadata);
                if (lr != null && lr.getListing() != null) {
                    for (final String file : lr.getListing()) {
                        result.add(new StoreResource(loc, path, file));
                    }
                }
            } catch (final TransferLocationException e) {
                fireIndyStoreErrorEvent(e);
                logger.warn("Location Error: " + e.getMessage(), e);
                throw new IndyWorkflowException("Failed to list path: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
            } catch (final TransferException e) {
                logger.error(e.getMessage(), e);
                throw new IndyWorkflowException("Failed to list path: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
            }
        } else {
            try {
                final ListingResult listing = transfers.list(res, eventMetadata);
                if (listing != null && listing.getListing() != null) {
                    for (final String child : listing.getListing()) {
                        result.add(new StoreResource(loc, path, child));
                    }
                }
            } catch (final TransferLocationException e) {
                fireIndyStoreErrorEvent(e);
                logger.warn("Timeout  / bad gateway: " + e.getMessage(), e);
                throw new IndyWorkflowException("Failed to list path: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
            } catch (final TransferException e) {
                logger.error(e.getMessage(), e);
                throw new IndyWorkflowException("Failed to list path: {} from: {}. Reason: {}", e, path, store.getKey(), e.getMessage());
            }
        }
    }
    return dedupeListing(result);
}
Also used : StoreResource(org.commonjava.indy.content.StoreResource) TransferException(org.commonjava.maven.galley.TransferException) KeyedLocation(org.commonjava.indy.model.galley.KeyedLocation) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) ArrayList(java.util.ArrayList) VirtualResource(org.commonjava.maven.galley.model.VirtualResource) RemoteRepository(org.commonjava.indy.model.core.RemoteRepository) ListingResult(org.commonjava.maven.galley.model.ListingResult) TransferLocationException(org.commonjava.maven.galley.TransferLocationException) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Aggregations

StoreResource (org.commonjava.indy.content.StoreResource)24 ArrayList (java.util.ArrayList)16 IndyWorkflowException (org.commonjava.indy.IndyWorkflowException)13 KeyedLocation (org.commonjava.indy.model.galley.KeyedLocation)10 HashMap (java.util.HashMap)9 EventMetadata (org.commonjava.maven.galley.event.EventMetadata)8 List (java.util.List)7 RemoteRepository (org.commonjava.indy.model.core.RemoteRepository)7 ConcreteResource (org.commonjava.maven.galley.model.ConcreteResource)7 ListingResult (org.commonjava.maven.galley.model.ListingResult)7 Set (java.util.Set)6 HashSet (java.util.HashSet)5 TransferException (org.commonjava.maven.galley.TransferException)5 ArtifactPathInfo (org.commonjava.atlas.maven.ident.util.ArtifactPathInfo)4 TransferLocationException (org.commonjava.maven.galley.TransferLocationException)4 Transfer (org.commonjava.maven.galley.model.Transfer)4 VirtualResource (org.commonjava.maven.galley.model.VirtualResource)4 Measure (org.commonjava.o11yphant.metrics.annotation.Measure)4 Test (org.junit.Test)4 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3