Search in sources :

Example 1 with Measure

use of org.commonjava.indy.measure.annotation.Measure in project indy by Commonjava.

the class KojiMavenMetadataProvider method getMetadata.

@Override
@IndyMetrics(measure = @Measure(timers = @MetricNamed(name = IndyMetricsKojiNames.METHOD_MAVENMETADATA_GETMETADATA + IndyMetricsNames.TIMER), meters = @MetricNamed(name = IndyMetricsKojiNames.METHOD_MAVENMETADATA_GETMETADATA + IndyMetricsNames.METER)))
public Metadata getMetadata(StoreKey targetKey, String path) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    if (group != targetKey.getType()) {
        logger.debug("Not a group. Cannot supplement with metadata from Koji builds");
        return null;
    }
    if (!kojiConfig.isEnabled()) {
        logger.debug("Koji add-on is disabled.");
        return null;
    }
    if (!kojiConfig.isEnabledFor(targetKey.getName())) {
        logger.debug("Koji integration is not enabled for group: {}", targetKey);
        return null;
    }
    File mdFile = new File(path);
    File artifactDir = mdFile.getParentFile();
    File groupDir = artifactDir == null ? null : artifactDir.getParentFile();
    if (artifactDir == null || groupDir == null) {
        logger.debug("Invalid groupId / artifactId directory structure: '{}' / '{}'", groupDir, artifactDir);
        return null;
    }
    String groupId = groupDir.getPath().replace(File.separatorChar, '.');
    String artifactId = artifactDir.getName();
    ProjectRef ga = null;
    try {
        ga = new SimpleProjectRef(groupId, artifactId);
    } catch (InvalidRefException e) {
        logger.debug("Not a valid Maven GA: {}:{}. Skipping Koji metadata retrieval.", groupId, artifactId);
    }
    if (ga == null) {
        logger.debug("Could not render a valid Maven GA for path: '{}'", path);
        return null;
    }
    ReentrantLock lock;
    synchronized (versionMetadataLocks) {
        lock = versionMetadataLocks.get(ga);
        if (lock == null) {
            lock = new ReentrantLock();
            versionMetadataLocks.put(ga, lock);
        }
    }
    boolean locked = false;
    try {
        locked = lock.tryLock(kojiConfig.getLockTimeoutSeconds(), TimeUnit.SECONDS);
        if (!locked) {
            throw new IndyWorkflowException("Failed to acquire Koji GA version metadata lock on: %s in %d seconds.", ga, kojiConfig.getLockTimeoutSeconds());
        }
        Metadata metadata = versionMetadata.get(ga);
        ProjectRef ref = ga;
        if (metadata == null) {
            try {
                metadata = kojiClient.withKojiSession((session) -> {
                    // short-term caches to help improve performance a bit by avoiding xml-rpc calls.
                    Map<Integer, KojiBuildArchiveCollection> seenBuildArchives = new HashMap<>();
                    Set<Integer> seenBuilds = new HashSet<>();
                    List<KojiArchiveInfo> archives = kojiClient.listArchivesMatching(ref, session);
                    Set<SingleVersion> versions = new HashSet<>();
                    for (KojiArchiveInfo archive : archives) {
                        if (!archive.getFilename().endsWith(".pom")) {
                            logger.debug("Skipping non-POM: {}", archive.getFilename());
                            continue;
                        }
                        SingleVersion singleVersion = VersionUtils.createSingleVersion(archive.getVersion());
                        if (versions.contains(singleVersion)) {
                            logger.debug("Skipping already collected version: {}", archive.getVersion());
                            continue;
                        }
                        KojiBuildInfo build;
                        if (seenBuilds.contains(archive.getBuildId())) {
                            logger.debug("Skipping already seen build: {}", archive.getBuildId());
                            continue;
                        } else {
                            build = kojiClient.getBuildInfo(archive.getBuildId(), session);
                            seenBuilds.add(archive.getBuildId());
                        }
                        if (build == null) {
                            logger.debug("Cannot retrieve build info: {}. Skipping: {}", archive.getBuildId(), archive.getFilename());
                            continue;
                        }
                        if (build.getBuildState() != KojiBuildState.COMPLETE) {
                            logger.debug("Build: {} is not completed. The state is {}. Skipping.", build.getNvr(), build.getBuildState());
                            continue;
                        }
                        if (build.getTaskId() == null) {
                            logger.debug("Build: {} is not a real build. It looks like a binary import. Skipping.", build.getNvr());
                            // This is not a real build, it's a binary import.
                            continue;
                        }
                        boolean buildAllowed = false;
                        if (!kojiConfig.isTagPatternsEnabled()) {
                            buildAllowed = true;
                        } else {
                            logger.debug("Checking for builds/tags of: {}", archive);
                            List<KojiTagInfo> tags = kojiClient.listTags(build.getId(), session);
                            for (KojiTagInfo tag : tags) {
                                if (kojiConfig.isTagAllowed(tag.getName())) {
                                    logger.debug("Koji tag: {} is allowed for proxying.", tag.getName());
                                    buildAllowed = true;
                                    break;
                                } else {
                                    logger.debug("Koji tag: {} is not allowed for proxying.", tag.getName());
                                }
                            }
                        }
                        logger.debug("Checking if build passed tag whitelist check and doesn't collide with something in authority store (if configured)...");
                        if (buildAllowed && buildAuthority.isAuthorized(path, new EventMetadata(), ref, build, session, seenBuildArchives)) {
                            try {
                                logger.debug("Adding version: {} for: {}", archive.getVersion(), path);
                                versions.add(singleVersion);
                            } catch (InvalidVersionSpecificationException e) {
                                logger.warn(String.format("Encountered invalid version: %s for archive: %s. Reason: %s", archive.getVersion(), archive.getArchiveId(), e.getMessage()), e);
                            }
                        }
                    }
                    if (versions.isEmpty()) {
                        logger.debug("No versions found in Koji builds for metadata: {}", path);
                        return null;
                    }
                    List<SingleVersion> sortedVersions = new ArrayList<>(versions);
                    Collections.sort(sortedVersions);
                    Metadata md = new Metadata();
                    md.setGroupId(ref.getGroupId());
                    md.setArtifactId(ref.getArtifactId());
                    Versioning versioning = new Versioning();
                    versioning.setRelease(sortedVersions.get(versions.size() - 1).renderStandard());
                    versioning.setLatest(sortedVersions.get(versions.size() - 1).renderStandard());
                    versioning.setVersions(sortedVersions.stream().map(SingleVersion::renderStandard).collect(Collectors.toList()));
                    Date lastUpdated = Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime();
                    versioning.setLastUpdated(new SimpleDateFormat(LAST_UPDATED_FORMAT).format(lastUpdated));
                    md.setVersioning(versioning);
                    return md;
                });
            } catch (KojiClientException e) {
                throw new IndyWorkflowException("Failed to retrieve version metadata for: %s from Koji. Reason: %s", e, ga, e.getMessage());
            }
            Metadata md = metadata;
            if (metadata != null) {
                // FIXME: Need a way to listen for cache expiration and re-request this?
                versionMetadata.execute((cache) -> cache.getAdvancedCache().put(ref, md, kojiConfig.getMetadataTimeoutSeconds(), TimeUnit.SECONDS));
            }
        }
        return metadata;
    } catch (InterruptedException e) {
        logger.warn("Interrupted waiting for Koji GA version metadata lock on target: {}", ga);
    } finally {
        if (locked) {
            lock.unlock();
        }
    }
    logger.debug("Returning null metadata result for unknown reason (path: '{}')", path);
    return null;
}
Also used : SingleVersion(org.commonjava.maven.atlas.ident.version.SingleVersion) Date(java.util.Date) SimpleProjectRef(org.commonjava.maven.atlas.ident.ref.SimpleProjectRef) LoggerFactory(org.slf4j.LoggerFactory) MavenMetadataProvider(org.commonjava.indy.pkg.maven.content.group.MavenMetadataProvider) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) KojiTagInfo(com.redhat.red.build.koji.model.xmlrpc.KojiTagInfo) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) VersionUtils(org.commonjava.maven.atlas.ident.util.VersionUtils) Calendar(java.util.Calendar) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) MetricNamed(org.commonjava.indy.measure.annotation.MetricNamed) ProjectRef(org.commonjava.maven.atlas.ident.ref.ProjectRef) Map(java.util.Map) KojiArchiveInfo(com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo) KojiClientException(com.redhat.red.build.koji.KojiClientException) StoreKey(org.commonjava.indy.model.core.StoreKey) WeakHashMap(java.util.WeakHashMap) StoreType.group(org.commonjava.indy.model.core.StoreType.group) Versioning(org.apache.maven.artifact.repository.metadata.Versioning) Logger(org.slf4j.Logger) ReentrantLock(java.util.concurrent.locks.ReentrantLock) TimeZone(java.util.TimeZone) KojiClient(com.redhat.red.build.koji.KojiClient) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics) Set(java.util.Set) Collectors(java.util.stream.Collectors) CacheHandle(org.commonjava.indy.subsys.infinispan.CacheHandle) InvalidRefException(org.commonjava.maven.atlas.ident.ref.InvalidRefException) File(java.io.File) IndyMetricsKojiNames(org.commonjava.indy.koji.metrics.IndyMetricsKojiNames) TimeUnit(java.util.concurrent.TimeUnit) KojiBuildState(com.redhat.red.build.koji.model.xmlrpc.KojiBuildState) InvalidVersionSpecificationException(org.commonjava.maven.atlas.ident.version.InvalidVersionSpecificationException) List(java.util.List) KojiMavenVersionMetadataCache(org.commonjava.indy.koji.inject.KojiMavenVersionMetadataCache) Measure(org.commonjava.indy.measure.annotation.Measure) KojiBuildInfo(com.redhat.red.build.koji.model.xmlrpc.KojiBuildInfo) IndyMetricsNames(org.commonjava.indy.IndyMetricsNames) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) Metadata(org.apache.maven.artifact.repository.metadata.Metadata) ApplicationScoped(javax.enterprise.context.ApplicationScoped) KojiBuildArchiveCollection(com.redhat.red.build.koji.model.xmlrpc.KojiBuildArchiveCollection) Collections(java.util.Collections) IndyKojiConfig(org.commonjava.indy.koji.conf.IndyKojiConfig) HashSet(java.util.HashSet) Set(java.util.Set) InvalidRefException(org.commonjava.maven.atlas.ident.ref.InvalidRefException) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) Metadata(org.apache.maven.artifact.repository.metadata.Metadata) Logger(org.slf4j.Logger) KojiTagInfo(com.redhat.red.build.koji.model.xmlrpc.KojiTagInfo) KojiArchiveInfo(com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) ArrayList(java.util.ArrayList) List(java.util.List) SingleVersion(org.commonjava.maven.atlas.ident.version.SingleVersion) KojiBuildInfo(com.redhat.red.build.koji.model.xmlrpc.KojiBuildInfo) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Date(java.util.Date) SimpleProjectRef(org.commonjava.maven.atlas.ident.ref.SimpleProjectRef) Versioning(org.apache.maven.artifact.repository.metadata.Versioning) InvalidVersionSpecificationException(org.commonjava.maven.atlas.ident.version.InvalidVersionSpecificationException) KojiClientException(com.redhat.red.build.koji.KojiClientException) SimpleProjectRef(org.commonjava.maven.atlas.ident.ref.SimpleProjectRef) ProjectRef(org.commonjava.maven.atlas.ident.ref.ProjectRef) File(java.io.File) HashMap(java.util.HashMap) Map(java.util.Map) WeakHashMap(java.util.WeakHashMap) SimpleDateFormat(java.text.SimpleDateFormat) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics)

Example 2 with Measure

use of org.commonjava.indy.measure.annotation.Measure in project indy by Commonjava.

the class MetricsInterceptor method operation.

@AroundInvoke
public Object operation(InvocationContext context) throws Exception {
    if (!config.isMetricsEnabled())
        return context.proceed();
    IndyMetrics metrics = context.getMethod().getAnnotation(IndyMetrics.class);
    if (metrics == null) {
        return context.proceed();
    }
    logger.debug("Gathering metrics for: {}", context.getContextData());
    Measure measures = metrics.measure();
    List<Timer.Context> timers = Stream.of(measures.timers()).map(named -> util.getTimer(named).time()).collect(Collectors.toList());
    try {
        return context.proceed();
    } catch (Exception e) {
        Measure me = metrics.exceptions();
        Stream.of(me.meters()).forEach((named) -> {
            Meter requests = util.getMeter(named);
            requests.mark();
        });
        throw e;
    } finally {
        if (timers != null) {
            timers.forEach(Timer.Context::stop);
        }
        Stream.of(measures.meters()).forEach((named) -> {
            Meter requests = util.getMeter(named);
            requests.mark();
        });
    }
}
Also used : Logger(org.slf4j.Logger) InvocationContext(javax.interceptor.InvocationContext) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics) IndyMetricsConfig(org.commonjava.indy.metrics.conf.IndyMetricsConfig) LoggerFactory(org.slf4j.LoggerFactory) Collectors(java.util.stream.Collectors) IndyMetricsManager(org.commonjava.indy.IndyMetricsManager) Inject(javax.inject.Inject) Interceptor(javax.interceptor.Interceptor) Meter(com.codahale.metrics.Meter) List(java.util.List) Measure(org.commonjava.indy.measure.annotation.Measure) Stream(java.util.stream.Stream) IndyMetricsNamed(org.commonjava.indy.metrics.conf.annotation.IndyMetricsNamed) Timer(com.codahale.metrics.Timer) AroundInvoke(javax.interceptor.AroundInvoke) InvocationContext(javax.interceptor.InvocationContext) Meter(com.codahale.metrics.Meter) Measure(org.commonjava.indy.measure.annotation.Measure) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics) AroundInvoke(javax.interceptor.AroundInvoke)

Example 3 with Measure

use of org.commonjava.indy.measure.annotation.Measure in project indy by Commonjava.

the class KojiBuildAuthority method isAuthorized.

@IndyMetrics(measure = @Measure(timers = @MetricNamed(name = IndyMetricsKojiNames.METHOD_BUILDAUTHORITY_ISAUTHORIZED + IndyMetricsNames.TIMER), meters = @MetricNamed(name = IndyMetricsKojiNames.METHOD_BUILDAUTHORITY_ISAUTHORIZED + IndyMetricsNames.METER)))
public boolean isAuthorized(String path, EventMetadata eventMetadata, ProjectRef ref, KojiBuildInfo build, KojiSessionInfo session, Map<Integer, KojiBuildArchiveCollection> seenBuildArchives) throws KojiClientException {
    ArtifactStore authoritativeStore = getAuthoritativeStore();
    if (authoritativeStore != null) {
        KojiBuildArchiveCollection archiveCollection = seenBuildArchives.get(build.getId());
        if (archiveCollection == null) {
            archiveCollection = kojiClient.listArchivesForBuild(build, session);
            seenBuildArchives.put(build.getId(), archiveCollection);
        }
        if (archiveCollection == null) {
            throw new KojiClientException("Failed to retrieve archives for build: %s", build);
        }
        // @formatter:off
        Predicate<KojiArchiveInfo> archiveInfoFilter = (archive) -> EXCLUDED_FILE_ENDINGS.stream().allMatch(ending -> !archive.getFilename().endsWith(ending));
        List<KojiArchiveInfo> sortedArchives = archiveCollection.getArchives().stream().filter(archiveInfoFilter).sorted((a1, a2) -> {
            TypePriority t1 = TypePriority.get(a1.getExtension());
            TypePriority t2 = TypePriority.get(a2.getExtension());
            return Integer.valueOf(t1.ordinal()).compareTo(t2.ordinal());
        }).collect(Collectors.toList());
        for (KojiArchiveInfo archive : sortedArchives) {
            try {
                if (isMavenArtifact(archive)) {
                    // skip non-Maven artifacts
                    continue;
                }
                if (containsPlaceholders(archive)) {
                    return false;
                }
                String artifactPath = ArtifactPathUtils.formatArtifactPath(archive.asArtifact(), typeMapper);
                String md5 = checksumArtifact(authoritativeStore, artifactPath, eventMetadata);
                if (isNotBlank(md5)) {
                    //FIXME: not sure if all koji archives are using md5 as checksum type for maven build
                    String kojiMd5 = archive.getChecksum();
                    Logger logger = LoggerFactory.getLogger(getClass());
                    logger.info("Checking checksum for {} (path: {}) in auth store {}, auth store checksum:{}, koji build check sum:{}", ref, path, authoritativeStore, md5, kojiMd5);
                    if (!md5.equals(kojiMd5)) {
                        // if checksum is not the same, it means the artifact in koji is DIFFERENT from the one in the authoritative store. Reject this.
                        return false;
                    }
                }
            } catch (Exception e) {
                Logger logger = LoggerFactory.getLogger(getClass());
                logger.error("SHOULD NEVER HAPPEN: Failed to transform artifact to path: " + e.getMessage(), e);
            }
        }
    }
    return true;
}
Also used : Arrays(java.util.Arrays) ContentDigester(org.commonjava.indy.content.ContentDigester) LoggerFactory(org.slf4j.LoggerFactory) TransferMetadata(org.commonjava.maven.galley.io.checksum.TransferMetadata) HashMap(java.util.HashMap) Inject(javax.inject.Inject) Transfer(org.commonjava.maven.galley.model.Transfer) FORCE_CHECKSUM(org.commonjava.maven.galley.io.ChecksummingTransferDecorator.FORCE_CHECKSUM) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) MetricNamed(org.commonjava.indy.measure.annotation.MetricNamed) ProjectRef(org.commonjava.maven.atlas.ident.ref.ProjectRef) Map(java.util.Map) StringUtils.trimToEmpty(org.apache.commons.lang.StringUtils.trimToEmpty) ContentDigest(org.commonjava.maven.galley.io.checksum.ContentDigest) KojiArchiveInfo(com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo) KojiClientException(com.redhat.red.build.koji.KojiClientException) StoreKey(org.commonjava.indy.model.core.StoreKey) ArtifactPathUtils(org.commonjava.maven.galley.maven.util.ArtifactPathUtils) Logger(org.slf4j.Logger) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) Predicate(java.util.function.Predicate) KojiClient(com.redhat.red.build.koji.KojiClient) StringUtils.contains(org.apache.commons.lang.StringUtils.contains) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) StringUtils.isNotBlank(org.apache.commons.lang.StringUtils.isNotBlank) IndyMetricsKojiNames(org.commonjava.indy.koji.metrics.IndyMetricsKojiNames) IOUtils(org.apache.commons.io.IOUtils) KojiSessionInfo(com.redhat.red.build.koji.model.xmlrpc.KojiSessionInfo) List(java.util.List) Measure(org.commonjava.indy.measure.annotation.Measure) KojiBuildInfo(com.redhat.red.build.koji.model.xmlrpc.KojiBuildInfo) IndyMetricsNames(org.commonjava.indy.IndyMetricsNames) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) TypeMapper(org.commonjava.maven.galley.maven.spi.type.TypeMapper) ApplicationScoped(javax.enterprise.context.ApplicationScoped) KojiBuildArchiveCollection(com.redhat.red.build.koji.model.xmlrpc.KojiBuildArchiveCollection) Collections(java.util.Collections) TRUE(java.lang.Boolean.TRUE) DirectContentAccess(org.commonjava.indy.content.DirectContentAccess) IndyDataException(org.commonjava.indy.data.IndyDataException) StoreDataManager(org.commonjava.indy.data.StoreDataManager) IndyKojiConfig(org.commonjava.indy.koji.conf.IndyKojiConfig) InputStream(java.io.InputStream) KojiBuildArchiveCollection(com.redhat.red.build.koji.model.xmlrpc.KojiBuildArchiveCollection) KojiClientException(com.redhat.red.build.koji.KojiClientException) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) Logger(org.slf4j.Logger) KojiArchiveInfo(com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) KojiClientException(com.redhat.red.build.koji.KojiClientException) IOException(java.io.IOException) IndyDataException(org.commonjava.indy.data.IndyDataException) IndyMetrics(org.commonjava.indy.measure.annotation.IndyMetrics)

Aggregations

List (java.util.List)3 Collectors (java.util.stream.Collectors)3 Inject (javax.inject.Inject)3 IndyMetrics (org.commonjava.indy.measure.annotation.IndyMetrics)3 Measure (org.commonjava.indy.measure.annotation.Measure)3 KojiClient (com.redhat.red.build.koji.KojiClient)2 KojiClientException (com.redhat.red.build.koji.KojiClientException)2 KojiArchiveInfo (com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo)2 KojiBuildArchiveCollection (com.redhat.red.build.koji.model.xmlrpc.KojiBuildArchiveCollection)2 KojiBuildInfo (com.redhat.red.build.koji.model.xmlrpc.KojiBuildInfo)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 ApplicationScoped (javax.enterprise.context.ApplicationScoped)2 IndyMetricsNames (org.commonjava.indy.IndyMetricsNames)2 IndyWorkflowException (org.commonjava.indy.IndyWorkflowException)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 Meter (com.codahale.metrics.Meter)1 Timer (com.codahale.metrics.Timer)1