Examples with EventMetadata org.commonjava.maven.galley.event.EventMetadata used on opensource projects

Example 16 with EventMetadata

use of org.commonjava.maven.galley.event.EventMetadata in project indy by Commonjava.

the class PromotionManager method runPathPromotions.

private PathsPromoteResult runPathPromotions(final PathsPromoteRequest request, final Set<String> pending, final Set<String> prevComplete, final Set<String> prevSkipped, final List<Transfer> contents, ValidationResult validation) {
    if (pending == null || pending.isEmpty()) {
        return new PathsPromoteResult(request, pending, prevComplete, prevSkipped, validation);
    }
    StoreKey targetKey = request.getTarget();
    ReentrantLock lock;
    synchronized (byPathTargetLocks) {
        lock = byPathTargetLocks.get(targetKey);
        if (lock == null) {
            lock = new ReentrantLock();
            byPathTargetLocks.put(targetKey, lock);
        }
    }
    final Set<String> complete = prevComplete == null ? new HashSet<>() : new HashSet<>(prevComplete);
    final Set<String> skipped = prevSkipped == null ? new HashSet<>() : new HashSet<>(prevSkipped);
    List<String> errors = new ArrayList<>();
    boolean locked = false;
    try {
        ArtifactStore sourceStore = storeManager.getArtifactStore(request.getSource());
        ArtifactStore targetStore = storeManager.getArtifactStore(request.getTarget());
        if (errors.isEmpty()) {
            locked = lock.tryLock(config.getLockTimeoutSeconds(), TimeUnit.SECONDS);
            if (!locked) {
                String error = String.format("Failed to acquire promotion lock on target: %s in %d seconds.", targetKey, config.getLockTimeoutSeconds());
                errors.add(error);
                logger.warn(error);
            }
        }
        if (errors.isEmpty()) {
            logger.info("Running promotions from: {} (key: {})\n  to: {} (key: {})", sourceStore, request.getSource(), targetStore, request.getTarget());
            final boolean purgeSource = request.isPurgeSource();
            contents.forEach((transfer) -> {
                try {
                    final String path = transfer.getPath();
                    Transfer target = contentManager.getTransfer(targetStore, path, TransferOperation.UPLOAD);
                    // TODO: Should the request object have an overwrite attribute? Is that something the user is qualified to decide?
                    if (target != null && target.exists()) {
                        logger.warn("NOT promoting: {} from: {} to: {}. Target file already exists.", path, request.getSource(), request.getTarget());
                        // TODO: There's no guarantee that the pre-existing content is the same!
                        pending.remove(path);
                        skipped.add(path);
                    } else {
                        try (InputStream stream = transfer.openInputStream(true)) {
                            contentManager.store(targetStore, path, stream, TransferOperation.UPLOAD, new EventMetadata());
                            pending.remove(path);
                            complete.add(path);
                            stream.close();
                            if (purgeSource) {
                                contentManager.delete(sourceStore, path, new EventMetadata());
                            }
                        } catch (final IOException e) {
                            String msg = String.format("Failed to open input stream for: %s. Reason: %s", transfer, e.getMessage());
                            errors.add(msg);
                            logger.error(msg, e);
                        }
                    }
                } catch (final IndyWorkflowException e) {
                    String msg = String.format("Failed to promote path: %s to: %s. Reason: %s", transfer, targetStore, e.getMessage());
                    errors.add(msg);
                    logger.error(msg, e);
                }
            });
        }
    } catch (InterruptedException e) {
        String error = String.format("Interrupted waiting for promotion lock on target: %s", targetKey);
        errors.add(error);
        logger.warn(error);
    } catch (final IndyDataException e) {
        String msg = String.format("Failed to retrieve artifact store: %s. Reason: %s", request.getSource(), e.getMessage());
        errors.add(msg);
        logger.error(msg, e);
    } finally {
        if (locked) {
            lock.unlock();
        }
    }
    String error = null;
    if (!errors.isEmpty()) {
        error = StringUtils.join(errors, "\n");
    }
    return new PathsPromoteResult(request, pending, complete, skipped, error, validation);
}

Example 17 with EventMetadata

use of org.commonjava.maven.galley.event.EventMetadata in project indy by Commonjava.

the class PromotionManagerTest method promoteAllByPath_PushTwoArtifactsToHostedRepo_VerifyCopiedToOtherHostedRepo.

@Test
public void promoteAllByPath_PushTwoArtifactsToHostedRepo_VerifyCopiedToOtherHostedRepo() throws Exception {
    final HostedRepository source = new HostedRepository(MAVEN_PKG_KEY, "source");
    storeManager.storeArtifactStore(source, new ChangeSummary(ChangeSummary.SYSTEM_USER, "test setup"), false, true, new EventMetadata());
    final String first = "/first/path";
    final String second = "/second/path";
    contentManager.store(source, first, new ByteArrayInputStream("This is a test".getBytes()), TransferOperation.UPLOAD, new EventMetadata());
    contentManager.store(source, second, new ByteArrayInputStream("This is a test".getBytes()), TransferOperation.UPLOAD, new EventMetadata());
    final HostedRepository target = new HostedRepository(MAVEN_PKG_KEY, "target");
    storeManager.storeArtifactStore(target, new ChangeSummary(ChangeSummary.SYSTEM_USER, "test setup"), false, true, new EventMetadata());
    final PathsPromoteResult result = manager.promotePaths(new PathsPromoteRequest(source.getKey(), target.getKey()), FAKE_BASE_URL);
    assertThat(result.getRequest().getSource(), equalTo(source.getKey()));
    assertThat(result.getRequest().getTarget(), equalTo(target.getKey()));
    final Set<String> pending = result.getPendingPaths();
    assertThat(pending == null || pending.isEmpty(), equalTo(true));
    final Set<String> completed = result.getCompletedPaths();
    assertThat(completed, notNullValue());
    assertThat(completed.size(), equalTo(2));
    assertThat(result.getError(), nullValue());
    Transfer ref = downloadManager.getStorageReference(target, first);
    assertThat(ref.exists(), equalTo(true));
    ref = downloadManager.getStorageReference(target, second);
    assertThat(ref.exists(), equalTo(true));
}

Example 18 with EventMetadata

use of org.commonjava.maven.galley.event.EventMetadata in project indy by Commonjava.

the class KojiContentManagerDecorator method getTransfer.

@Override
public Transfer getTransfer(final ArtifactStore store, final String path, final TransferOperation operation) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    logger.info("KOJI: Delegating initial getTransfer() attempt for: {}/{}", store.getKey(), path);
    Transfer result = delegate.getTransfer(store, path, operation);
    if (result == null && TransferOperation.DOWNLOAD == operation && StoreType.group == store.getKey().getType()) {
        logger.info("KOJI: Checking for Koji build matching: {}", path);
        Group group = (Group) store;
        RemoteRepository kojiProxy = findKojiBuildAnd(store, path, new EventMetadata(), null, this::createRemoteRepository);
        if (kojiProxy != null) {
            adjustTargetGroup(kojiProxy, group);
            EventMetadata eventMetadata = new EventMetadata().set(ContentManager.ENTRY_POINT_STORE, store.getKey());
            result = delegate.retrieve(kojiProxy, path, eventMetadata);
        }
        if (result != null && result.exists()) {
            nfc.clearMissing(new ConcreteResource(LocationUtils.toLocation(store), path));
        }
    }
    // Finally, pass the Transfer back.
    return result;
}

Example 19 with EventMetadata

use of org.commonjava.maven.galley.event.EventMetadata in project indy by Commonjava.

the class KojiContentManagerDecorator method createRemoteRepository.

private RemoteRepository createRemoteRepository(StoreKey inStore, ArtifactRef artifactRef, final KojiBuildInfo build, final KojiSessionInfo session) throws KojiClientException {
    Logger logger = LoggerFactory.getLogger(getClass());
    try {
        KojiBuildArchiveCollection archiveCollection = kojiClient.listArchivesForBuild(build, session);
        boolean isBinaryBuild = isBinaryBuild(build);
        String name = getRepositoryName(build, isBinaryBuild);
        // Using a RemoteRepository allows us to use the higher-level APIs in Indy, as opposed to TransferManager
        final KojiRepositoryCreator creator = createRepoCreator();
        if (creator == null) {
            throw new KojiClientException("Cannot proceed without a valid KojiRepositoryCreator instance.");
        }
        RemoteRepository remote = creator.createRemoteRepository(inStore.getPackageType(), name, formatStorageUrl(build), config.getDownloadTimeoutSeconds());
        remote.setServerCertPem(config.getServerPemContent());
        if (isBinaryBuild) {
            remote.setMetadata(ArtifactStore.METADATA_ORIGIN, KOJI_ORIGIN_BINARY);
        } else {
            remote.setMetadata(ArtifactStore.METADATA_ORIGIN, KOJI_ORIGIN);
        }
        // set pathMaskPatterns using build output paths
        Set<String> patterns = new HashSet<>();
        patterns.addAll(archiveCollection.getArchives().stream().map(a -> a.getGroupId().replace('.', '/') + "/" + a.getArtifactId() + "/" + a.getVersion() + "/" + a.getFilename()).collect(Collectors.toSet()));
        remote.setPathMaskPatterns(patterns);
        remote.setMetadata(CREATION_TRIGGER_GAV, artifactRef.toString());
        remote.setMetadata(NVR, build.getNvr());
        final ChangeSummary changeSummary = new ChangeSummary(ChangeSummary.SYSTEM_USER, "Creating remote repository for Koji build: " + build.getNvr());
        storeDataManager.storeArtifactStore(remote, changeSummary, false, true, new EventMetadata());
        logger.debug("Koji {}, add pathMaskPatterns: {}", name, patterns);
        return remote;
    } catch (MalformedURLException e) {
        throw new KojiClientException("Koji add-on seems misconifigured. Could not generate URL to repo for " + "build: %s\nBase URL: %s\nError: %s", e, build.getNvr(), config.getStorageRootUrl(), e.getMessage());
    } catch (IOException e) {
        throw new KojiClientException("Failed to read server SSL PEM information from Koji configuration for new hosted repo: %s", e, e.getMessage());
    } catch (IndyDataException e) {
        throw new KojiClientException("Failed to store temporary remote repo: %s", e, e.getMessage());
    }
}

Example 20 with EventMetadata

use of org.commonjava.maven.galley.event.EventMetadata in project indy by Commonjava.

the class KojiMavenMetadataProvider method getMetadata.

@Override
@IndyMetrics(measure = @Measure(timers = @MetricNamed(name = IndyMetricsKojiNames.METHOD_MAVENMETADATA_GETMETADATA + IndyMetricsNames.TIMER), meters = @MetricNamed(name = IndyMetricsKojiNames.METHOD_MAVENMETADATA_GETMETADATA + IndyMetricsNames.METER)))
public Metadata getMetadata(StoreKey targetKey, String path) throws IndyWorkflowException {
    Logger logger = LoggerFactory.getLogger(getClass());
    if (group != targetKey.getType()) {
        logger.debug("Not a group. Cannot supplement with metadata from Koji builds");
        return null;
    }
    if (!kojiConfig.isEnabled()) {
        logger.debug("Koji add-on is disabled.");
        return null;
    }
    if (!kojiConfig.isEnabledFor(targetKey.getName())) {
        logger.debug("Koji integration is not enabled for group: {}", targetKey);
        return null;
    }
    File mdFile = new File(path);
    File artifactDir = mdFile.getParentFile();
    File groupDir = artifactDir == null ? null : artifactDir.getParentFile();
    if (artifactDir == null || groupDir == null) {
        logger.debug("Invalid groupId / artifactId directory structure: '{}' / '{}'", groupDir, artifactDir);
        return null;
    }
    String groupId = groupDir.getPath().replace(File.separatorChar, '.');
    String artifactId = artifactDir.getName();
    ProjectRef ga = null;
    try {
        ga = new SimpleProjectRef(groupId, artifactId);
    } catch (InvalidRefException e) {
        logger.debug("Not a valid Maven GA: {}:{}. Skipping Koji metadata retrieval.", groupId, artifactId);
    }
    if (ga == null) {
        logger.debug("Could not render a valid Maven GA for path: '{}'", path);
        return null;
    }
    ReentrantLock lock;
    synchronized (versionMetadataLocks) {
        lock = versionMetadataLocks.get(ga);
        if (lock == null) {
            lock = new ReentrantLock();
            versionMetadataLocks.put(ga, lock);
        }
    }
    boolean locked = false;
    try {
        locked = lock.tryLock(kojiConfig.getLockTimeoutSeconds(), TimeUnit.SECONDS);
        if (!locked) {
            throw new IndyWorkflowException("Failed to acquire Koji GA version metadata lock on: %s in %d seconds.", ga, kojiConfig.getLockTimeoutSeconds());
        }
        Metadata metadata = versionMetadata.get(ga);
        ProjectRef ref = ga;
        if (metadata == null) {
            try {
                metadata = kojiClient.withKojiSession((session) -> {
                    // short-term caches to help improve performance a bit by avoiding xml-rpc calls.
                    Map<Integer, KojiBuildArchiveCollection> seenBuildArchives = new HashMap<>();
                    Set<Integer> seenBuilds = new HashSet<>();
                    List<KojiArchiveInfo> archives = kojiClient.listArchivesMatching(ref, session);
                    Set<SingleVersion> versions = new HashSet<>();
                    for (KojiArchiveInfo archive : archives) {
                        if (!archive.getFilename().endsWith(".pom")) {
                            logger.debug("Skipping non-POM: {}", archive.getFilename());
                            continue;
                        }
                        SingleVersion singleVersion = VersionUtils.createSingleVersion(archive.getVersion());
                        if (versions.contains(singleVersion)) {
                            logger.debug("Skipping already collected version: {}", archive.getVersion());
                            continue;
                        }
                        KojiBuildInfo build;
                        if (seenBuilds.contains(archive.getBuildId())) {
                            logger.debug("Skipping already seen build: {}", archive.getBuildId());
                            continue;
                        } else {
                            build = kojiClient.getBuildInfo(archive.getBuildId(), session);
                            seenBuilds.add(archive.getBuildId());
                        }
                        if (build == null) {
                            logger.debug("Cannot retrieve build info: {}. Skipping: {}", archive.getBuildId(), archive.getFilename());
                            continue;
                        }
                        if (build.getBuildState() != KojiBuildState.COMPLETE) {
                            logger.debug("Build: {} is not completed. The state is {}. Skipping.", build.getNvr(), build.getBuildState());
                            continue;
                        }
                        if (build.getTaskId() == null) {
                            logger.debug("Build: {} is not a real build. It looks like a binary import. Skipping.", build.getNvr());
                            // This is not a real build, it's a binary import.
                            continue;
                        }
                        boolean buildAllowed = false;
                        if (!kojiConfig.isTagPatternsEnabled()) {
                            buildAllowed = true;
                        } else {
                            logger.debug("Checking for builds/tags of: {}", archive);
                            List<KojiTagInfo> tags = kojiClient.listTags(build.getId(), session);
                            for (KojiTagInfo tag : tags) {
                                if (kojiConfig.isTagAllowed(tag.getName())) {
                                    logger.debug("Koji tag: {} is allowed for proxying.", tag.getName());
                                    buildAllowed = true;
                                    break;
                                } else {
                                    logger.debug("Koji tag: {} is not allowed for proxying.", tag.getName());
                                }
                            }
                        }
                        logger.debug("Checking if build passed tag whitelist check and doesn't collide with something in authority store (if configured)...");
                        if (buildAllowed && buildAuthority.isAuthorized(path, new EventMetadata(), ref, build, session, seenBuildArchives)) {
                            try {
                                logger.debug("Adding version: {} for: {}", archive.getVersion(), path);
                                versions.add(singleVersion);
                            } catch (InvalidVersionSpecificationException e) {
                                logger.warn(String.format("Encountered invalid version: %s for archive: %s. Reason: %s", archive.getVersion(), archive.getArchiveId(), e.getMessage()), e);
                            }
                        }
                    }
                    if (versions.isEmpty()) {
                        logger.debug("No versions found in Koji builds for metadata: {}", path);
                        return null;
                    }
                    List<SingleVersion> sortedVersions = new ArrayList<>(versions);
                    Collections.sort(sortedVersions);
                    Metadata md = new Metadata();
                    md.setGroupId(ref.getGroupId());
                    md.setArtifactId(ref.getArtifactId());
                    Versioning versioning = new Versioning();
                    versioning.setRelease(sortedVersions.get(versions.size() - 1).renderStandard());
                    versioning.setLatest(sortedVersions.get(versions.size() - 1).renderStandard());
                    versioning.setVersions(sortedVersions.stream().map(SingleVersion::renderStandard).collect(Collectors.toList()));
                    Date lastUpdated = Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime();
                    versioning.setLastUpdated(new SimpleDateFormat(LAST_UPDATED_FORMAT).format(lastUpdated));
                    md.setVersioning(versioning);
                    return md;
                });
            } catch (KojiClientException e) {
                throw new IndyWorkflowException("Failed to retrieve version metadata for: %s from Koji. Reason: %s", e, ga, e.getMessage());
            }
            Metadata md = metadata;
            if (metadata != null) {
                // FIXME: Need a way to listen for cache expiration and re-request this?
                versionMetadata.execute((cache) -> cache.getAdvancedCache().put(ref, md, kojiConfig.getMetadataTimeoutSeconds(), TimeUnit.SECONDS));
            }
        }
        return metadata;
    } catch (InterruptedException e) {
        logger.warn("Interrupted waiting for Koji GA version metadata lock on target: {}", ga);
    } finally {
        if (locked) {
            lock.unlock();
        }
    }
    logger.debug("Returning null metadata result for unknown reason (path: '{}')", path);
    return null;
}