Search in sources :

Example 16 with Measure

use of org.commonjava.o11yphant.metrics.annotation.Measure in project indy by Commonjava.

the class KojiMavenMetadataProvider method getMetadata.

@Override
@Measure
public Metadata getMetadata(StoreKey targetKey, String path) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    if (group != targetKey.getType()) {
        logger.debug("Not a group. Cannot supplement with metadata from Koji builds");
        return null;
    }
    if (!kojiConfig.isEnabled()) {
        logger.debug("Koji add-on is disabled.");
        return null;
    }
    try {
        ArtifactStore target = storeDataManager.getArtifactStore(targetKey);
        if (!kojiConfig.isEnabledFor(target)) {
            logger.debug("Koji integration is not enabled for group: {}", targetKey);
            return null;
        }
    } catch (IndyDataException e) {
        logger.error("Failed to get metadata for path {} in store {}: {}", path, targetKey, e.getMessage());
    }
    File mdFile = new File(path);
    File artifactDir = mdFile.getParentFile();
    File groupDir = artifactDir == null ? null : artifactDir.getParentFile();
    if (artifactDir == null || groupDir == null) {
        logger.debug("Invalid groupId / artifactId directory structure: '{}' / '{}'", groupDir, artifactDir);
        return null;
    }
    String groupId = groupDir.getPath().replace(File.separatorChar, '.');
    String artifactId = artifactDir.getName();
    ProjectRef ref = null;
    try {
        ref = new SimpleProjectRef(groupId, artifactId);
    } catch (InvalidRefException e) {
        logger.warn("Not a valid Maven GA: {}:{}. Skipping Koji metadata retrieval.", groupId, artifactId);
    }
    if (ref == null) {
        logger.debug("Could not render a valid Maven GA for path: '{}'", path);
        return null;
    }
    ProjectRef ga = ref;
    AtomicReference<IndyWorkflowException> wfError = new AtomicReference<>();
    return versionMetadataLocks.lockAnd(ga, kojiConfig.getLockTimeoutSeconds(), k -> {
        Metadata metadata = versionMetadata.get(ga);
        if (metadata == null) {
            try {
                metadata = executeKojiMetadataLookup(ga, path);
            } catch (IndyWorkflowException e) {
                wfError.set(e);
                metadata = null;
            } catch (KojiClientException e) {
                // FIXME: Should this bubble up like IndyWorkflowException does in the case of overloaded threadpool?
                Throwable cause = e.getCause();
                logger.error(String.format("Failed to retrieve version metadata for: %s from Koji. Reason: %s", ga, e.getMessage()), e);
                if (cause instanceof RuntimeException) {
                    logger.error("Previous exception's nested cause was a RuntimeException variant:", cause);
                }
                metadata = null;
            }
            if (metadata != null) {
                Metadata md = metadata;
                // FIXME: Need a way to listen for cache expiration and re-request this?
                versionMetadata.execute((cache) -> cache.put(ga, md, kojiConfig.getMetadataTimeoutSeconds(), TimeUnit.SECONDS));
            } else {
                logger.debug("Returning null metadata result for unknown reason (path: '{}')", path);
            }
        }
        return metadata;
    }, (k, lock) -> {
        logger.error("Failed to acquire Koji GA version metadata lock on: '{}' in {} seconds.", ga, kojiConfig.getLockTimeoutSeconds());
        return false;
    });
}
Also used : InvalidRefException(org.commonjava.atlas.maven.ident.ref.InvalidRefException) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) Metadata(org.apache.maven.artifact.repository.metadata.Metadata) AtomicReference(java.util.concurrent.atomic.AtomicReference) Logger(org.slf4j.Logger) IndyDataException(org.commonjava.indy.data.IndyDataException) SimpleProjectRef(org.commonjava.atlas.maven.ident.ref.SimpleProjectRef) KojiClientException(com.redhat.red.build.koji.KojiClientException) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) SimpleProjectRef(org.commonjava.atlas.maven.ident.ref.SimpleProjectRef) ProjectRef(org.commonjava.atlas.maven.ident.ref.ProjectRef) File(java.io.File) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 17 with Measure

use of org.commonjava.o11yphant.metrics.annotation.Measure in project indy by Commonjava.

the class DefaultContentManager method store.

@Override
@Measure
public Transfer store(final ArtifactStore store, final String path, final InputStream stream, final TransferOperation op, final EventMetadata eventMetadata) throws IndyWorkflowException {
    if (group == store.getKey().getType()) {
        try {
            final List<ArtifactStore> allMembers = storeManager.query().enabledState(true).getOrderedConcreteStoresInGroup(store.getPackageType(), store.getName());
            final Transfer txfr = store(allMembers, store.getKey(), path, stream, op, eventMetadata);
            logger.debug("Stored: {} for group: {} in: {}", path, store.getKey(), txfr);
            return txfr;
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Failed to lookup concrete members of: %s. Reason: %s", e, store, e.getMessage());
        }
    }
    logger.debug("Storing: {} for: {} with event metadata: {}", path, store.getKey(), eventMetadata);
    final Transfer txfr = downloadManager.store(store, path, stream, op, eventMetadata);
    if (txfr != null) {
        final KeyedLocation kl = (KeyedLocation) txfr.getLocation();
        ArtifactStore transferStore;
        try {
            transferStore = storeManager.getArtifactStore(kl.getKey());
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Failed to lookup store: %s. Reason: %s", e, kl.getKey(), e.getMessage());
        }
        contentGeneratorManager.handleContentStorage(transferStore, path, txfr, eventMetadata);
        final String name = String.format("ContentNFCClean-StoreSingle-store(%s)-path(%s)", store.getKey(), path);
        final String context = String.format("Class: %s, method: %s, store: %s, path: %s", this.getClass().getName(), "store", store.getKey(), path);
        storeManager.asyncGroupAffectedBy(new StoreDataManager.ContextualTask(name, context, () -> clearNFCEntries(kl, path, eventMetadata)));
    }
    return txfr;
}
Also used : IndyDataException(org.commonjava.indy.data.IndyDataException) KeyedLocation(org.commonjava.indy.model.galley.KeyedLocation) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) Transfer(org.commonjava.maven.galley.model.Transfer) StoreDataManager(org.commonjava.indy.data.StoreDataManager) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 18 with Measure

use of org.commonjava.o11yphant.metrics.annotation.Measure in project indy by Commonjava.

the class DefaultContentManager method retrieve.

@Override
@Measure
public Transfer retrieve(final ArtifactStore store, final String path, final EventMetadata eventMetadata) throws IndyWorkflowException {
    Transfer item;
    if (group == store.getKey().getType()) {
        List<ArtifactStore> members = getOrderedConcreteStoresAndFilter((Group) store, path);
        if (logger.isDebugEnabled()) {
            logger.debug("{} is a group. Attempting downloads from (in order):\n  {}", store.getKey(), StringUtils.join(members, "\n  "));
        }
        item = contentGeneratorManager.generateGroupFileContent((Group) store, members, path, eventMetadata);
        boolean generated = (item != null);
        if (!generated) {
            if (PathMaskChecker.checkMask(store, path)) {
                for (final ArtifactStore member : members) {
                    try {
                        item = doRetrieve(member, path, eventMetadata);
                    } catch (IndyWorkflowException e) {
                        logger.error("Failed to retrieve artifact from for path {} from {} in group {}, error is: {}", path, member, store, e.getMessage());
                    }
                    if (item != null) {
                        // get the item from the first member store
                        break;
                    }
                }
            }
        }
    } else {
        item = doRetrieve(store, path, eventMetadata);
    }
    if (item != null) {
        logger.info("Returning transfer {} from {}", item, store.getKey());
    } else {
        logger.trace("Not found path {} from {}", path, store.getKey());
    }
    return item;
}
Also used : Group(org.commonjava.indy.model.core.Group) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) Transfer(org.commonjava.maven.galley.model.Transfer) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 19 with Measure

use of org.commonjava.o11yphant.metrics.annotation.Measure in project indy by Commonjava.

the class DefaultContentManager method clearNFCEntries.

@Measure
protected void clearNFCEntries(final KeyedLocation kl, final String path, EventMetadata eventMetadata) {
    try {
        Set<Group> groups = storeManager.affectedBy(Arrays.asList(kl.getKey()), eventMetadata);
        groups.stream().map((g) -> new ConcreteResource(LocationUtils.toLocation(g), path)).forEach((cr) -> nfc.clearMissing(cr));
        nfc.clearMissing(new ConcreteResource(kl, path));
    } catch (IndyDataException e) {
        logger.error(String.format("Failed to clear NFC entries affected by upload of: %s to: %s. Reason: %s", path, kl.getKey(), e.getMessage()), e);
    }
}
Also used : Arrays(java.util.Arrays) HTTP_METADATA_EXT(org.commonjava.maven.galley.io.SpecialPathConstants.HTTP_METADATA_EXT) ContentDigester(org.commonjava.indy.content.ContentDigester) GroupRepositoryFilterManager(org.commonjava.indy.core.content.group.GroupRepositoryFilterManager) KeyedLocation(org.commonjava.indy.model.galley.KeyedLocation) CHECK_CACHE_ONLY(org.commonjava.indy.IndyContentConstants.CHECK_CACHE_ONLY) LoggerFactory(org.slf4j.LoggerFactory) ContentUtils.dedupeListing(org.commonjava.indy.util.ContentUtils.dedupeListing) StringUtils(org.apache.commons.lang3.StringUtils) SpecialPathInfo(org.commonjava.maven.galley.model.SpecialPathInfo) Group(org.commonjava.indy.model.core.Group) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Transfer(org.commonjava.maven.galley.model.Transfer) Measure(org.commonjava.o11yphant.metrics.annotation.Measure) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) TransferOperation(org.commonjava.maven.galley.model.TransferOperation) TransferManager(org.commonjava.maven.galley.TransferManager) CASCADE(org.commonjava.indy.IndyContentConstants.CASCADE) DownloadManager(org.commonjava.indy.content.DownloadManager) ApplicationStatus(org.commonjava.indy.util.ApplicationStatus) StoreKey(org.commonjava.indy.model.core.StoreKey) StoreType.group(org.commonjava.indy.model.core.StoreType.group) LocationUtils(org.commonjava.indy.util.LocationUtils) ConcreteResource(org.commonjava.maven.galley.model.ConcreteResource) Logger(org.slf4j.Logger) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyObjectMapper(org.commonjava.indy.model.core.io.IndyObjectMapper) Set(java.util.Set) SpecialPathManager(org.commonjava.maven.galley.spi.io.SpecialPathManager) IOException(java.io.IOException) NotFoundCache(org.commonjava.maven.galley.spi.nfc.NotFoundCache) Collectors(java.util.stream.Collectors) ContentManager(org.commonjava.indy.content.ContentManager) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) IGNORE_READONLY(org.commonjava.indy.data.StoreDataManager.IGNORE_READONLY) IndyConfiguration(org.commonjava.indy.conf.IndyConfiguration) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) StoreResource(org.commonjava.indy.content.StoreResource) StoreType.hosted(org.commonjava.indy.model.core.StoreType.hosted) IndyDataException(org.commonjava.indy.data.IndyDataException) StoreDataManager(org.commonjava.indy.data.StoreDataManager) HttpExchangeMetadata(org.commonjava.maven.galley.transport.htcli.model.HttpExchangeMetadata) InputStream(java.io.InputStream) IndyDataException(org.commonjava.indy.data.IndyDataException) Group(org.commonjava.indy.model.core.Group) ConcreteResource(org.commonjava.maven.galley.model.ConcreteResource) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Example 20 with Measure

use of org.commonjava.o11yphant.metrics.annotation.Measure in project indy by Commonjava.

the class DefaultContentManager method store.

@Override
@Measure
public Transfer store(final List<? extends ArtifactStore> stores, final StoreKey topKey, final String path, final InputStream stream, final TransferOperation op, final EventMetadata eventMetadata) throws IndyWorkflowException {
    logger.debug("Storing: {} in: {} with event metadata: {}", path, stores, eventMetadata);
    final Transfer txfr = downloadManager.store(stores, path, stream, op, eventMetadata);
    if (txfr != null) {
        final KeyedLocation kl = (KeyedLocation) txfr.getLocation();
        ArtifactStore transferStore;
        try {
            transferStore = storeManager.getArtifactStore(kl.getKey());
        } catch (final IndyDataException e) {
            throw new IndyWorkflowException("Failed to lookup store: %s. Reason: %s", e, kl.getKey(), e.getMessage());
        }
        contentGeneratorManager.handleContentStorage(transferStore, path, txfr, eventMetadata);
        final String name = String.format("ContentNFCClean-StoreList-location(%s)-path(%s)", kl, path);
        final String context = String.format("Class: %s, method: %s, location: %s, path: %s", this.getClass().getName(), "store-stores", kl, path);
        storeManager.asyncGroupAffectedBy(new StoreDataManager.ContextualTask(name, context, () -> clearNFCEntries(kl, path, eventMetadata)));
    }
    return txfr;
}
Also used : IndyDataException(org.commonjava.indy.data.IndyDataException) KeyedLocation(org.commonjava.indy.model.galley.KeyedLocation) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) Transfer(org.commonjava.maven.galley.model.Transfer) StoreDataManager(org.commonjava.indy.data.StoreDataManager) Measure(org.commonjava.o11yphant.metrics.annotation.Measure)

Aggregations

Measure (org.commonjava.o11yphant.metrics.annotation.Measure)65 StoreKey (org.commonjava.indy.model.core.StoreKey)23 ArtifactStore (org.commonjava.indy.model.core.ArtifactStore)19 Transfer (org.commonjava.maven.galley.model.Transfer)19 Logger (org.slf4j.Logger)19 IndyWorkflowException (org.commonjava.indy.IndyWorkflowException)16 KeyedLocation (org.commonjava.indy.model.galley.KeyedLocation)14 IndyDataException (org.commonjava.indy.data.IndyDataException)13 ArrayList (java.util.ArrayList)12 EventMetadata (org.commonjava.maven.galley.event.EventMetadata)11 HashSet (java.util.HashSet)9 Group (org.commonjava.indy.model.core.Group)8 ConcreteResource (org.commonjava.maven.galley.model.ConcreteResource)8 BoundStatement (com.datastax.driver.core.BoundStatement)6 StoreDataManager (org.commonjava.indy.data.StoreDataManager)6 StoreKey.fromString (org.commonjava.indy.model.core.StoreKey.fromString)6 Query (org.infinispan.query.dsl.Query)6 IOException (java.io.IOException)5 Set (java.util.Set)5 AtomicReference (java.util.concurrent.atomic.AtomicReference)5