Search in sources :

Example 1 with PathsPromoteCompleteEvent

use of org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent in project indy by Commonjava.

the class EventAuditListener method onPromoteComplete.

public void onPromoteComplete(@Observes final PromoteCompleteEvent event) {
    if (!eventAuditConfig.isEnabled()) {
        return;
    }
    if (event instanceof PathsPromoteCompleteEvent) {
        FileGroupingEvent fileGroupingEvent = new FileGroupingEvent(FileGroupingEventType.BY_PATH_PROMOTION);
        transformFileGroupingEvent(event, fileGroupingEvent);
        eventPublisher.publishFileGroupingEvent(fileGroupingEvent);
    } else {
        logger.trace("Unsupported grouping event: {}", event.getClass());
        return;
    }
}
Also used : PathsPromoteCompleteEvent(org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent) FileGroupingEvent(org.commonjava.auditquery.fileevent.FileGroupingEvent)

Example 2 with PathsPromoteCompleteEvent

use of org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent in project indy by Commonjava.

the class PromotionManager method runPathPromotions.

private PathsPromoteResult runPathPromotions(final PathsPromoteRequest request, final Set<String> pending, final List<Transfer> contents, final ValidationResult validation) {
    long begin = System.currentTimeMillis();
    PromotionHelper.PromotionRepoRetrievalResult checkResult = promotionHelper.checkAndRetrieveSourceAndTargetRepos(request);
    if (checkResult.hasErrors()) {
        return new PathsPromoteResult(request, pending, emptySet(), emptySet(), StringUtils.join(checkResult.errors, "\n"), validation);
    }
    final ArtifactStore targetStore = checkResult.targetStore;
    StoreKey targetKey = targetStore.getKey();
    logger.info("Run promotion from: {} to: {}, paths: {}", request.getSource(), targetKey, pending);
    Set<Group> affectedGroups;
    try {
        affectedGroups = storeManager.query().getGroupsAffectedBy(targetKey);
        logger.info("Calculate affected groups, target: {}, affected-groups: {}", targetKey, affectedGroups);
    } catch (IndyDataException e) {
        logger.error("Get affected groups failed", e);
        return new PathsPromoteResult(request, pending, emptySet(), emptySet(), "Get affected groups failed, " + e.getMessage(), validation);
    }
    DrainingExecutorCompletionService<Set<PathTransferResult>> svc = new DrainingExecutorCompletionService<>(transferService);
    int corePoolSize = transferService.getCorePoolSize();
    int size = contents.size();
    int batchSize = getParalleledBatchSize(size, corePoolSize);
    logger.info("Execute parallel on collection, size: {}, batch: {}", size, batchSize);
    Collection<Collection<Transfer>> batches = batch(contents, batchSize);
    final List<String> errors = new ArrayList<>();
    try {
        detectOverloadVoid(() -> batches.forEach(batch -> svc.submit(newPathPromotionsJob(batch, targetStore, request, affectedGroups))));
    } catch (IndyWorkflowException e) {
        // might be PoolOverloadException. Log it and continue to revert any completed paths
        String msg = String.format("Failed to submit all path promotion jobs. Error: %s", e.toString());
        logger.error(msg, e);
        errors.add(msg);
    }
    final Set<PathTransferResult> results = new HashSet<>();
    try {
        svc.drain(results::addAll);
    } catch (InterruptedException | ExecutionException e) {
        String msg = String.format("Error waiting for promotion of: %s to: %s", request.getSource(), request.getTarget());
        logger.error(msg, e);
        errors.add(msg);
    }
    final Set<String> completed = new HashSet<>();
    final Set<String> skipped = new HashSet<>();
    results.forEach(result -> {
        if (result.error != null) {
            errors.add(result.error);
        } else if (result.skipped) {
            skipped.add(result.path);
        } else {
            completed.add(result.path);
        }
    });
    PathsPromoteResult result;
    if (!errors.isEmpty()) {
        List<String> rollbackErrors = promotionHelper.deleteFromStore(completed, targetStore);
        errors.addAll(rollbackErrors);
        result = new PathsPromoteResult(request, pending, emptySet(), emptySet(), StringUtils.join(errors, "\n"), validation);
    } else {
        result = new PathsPromoteResult(request, emptySet(), completed, skipped, null, validation);
        final String name = String.format("PromoteNFCClean-method(%s)-source(%s)-target(%s)", "runPathPromotions", request.getSource(), targetStore.getKey());
        final String context = String.format("Class: %s, method: %s, source: %s, target: %s", this.getClass().getName(), "runPathPromotions", request.getSource(), targetStore.getKey());
        storeManager.asyncGroupAffectedBy(new StoreDataManager.ContextualTask(name, context, () -> promotionHelper.clearStoreNFC(completed, targetStore, affectedGroups)));
        if (request.isFireEvents()) {
            fireEvent(promoteCompleteEvent, new PathsPromoteCompleteEvent(result));
        }
    }
    logger.info("Promotion completed, promotionId: {}, timeInSeconds: {}", request.getPromotionId(), timeInSeconds(begin));
    return result;
}
Also used : GroupMembershipLocks(org.commonjava.indy.core.inject.GroupMembershipLocks) ValidationResult(org.commonjava.indy.promote.model.ValidationResult) ChangeSummary(org.commonjava.indy.audit.ChangeSummary) LoggerFactory(org.slf4j.LoggerFactory) PathsPromoteCompleteEvent(org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent) StringUtils(org.apache.commons.lang3.StringUtils) SpecialPathInfo(org.commonjava.maven.galley.model.SpecialPathInfo) WeftExecutorService(org.commonjava.cdi.util.weft.WeftExecutorService) Batcher.getParalleledBatchSize(org.commonjava.indy.promote.util.Batcher.getParalleledBatchSize) Future(java.util.concurrent.Future) WeftManaged(org.commonjava.cdi.util.weft.WeftManaged) Map(java.util.Map) StoreKey(org.commonjava.indy.model.core.StoreKey) PathsPromoteRequest(org.commonjava.indy.promote.model.PathsPromoteRequest) PromotionCallbackHelper(org.commonjava.indy.promote.callback.PromotionCallbackHelper) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StoreType(org.commonjava.indy.model.core.StoreType) Set(java.util.Set) ExceptionUtils.getStackTrace(org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace) Collectors(java.util.stream.Collectors) ContentManager(org.commonjava.indy.content.ContentManager) List(java.util.List) EventMetadata(org.commonjava.maven.galley.event.EventMetadata) PromotionHelper.throwProperException(org.commonjava.indy.promote.data.PromotionHelper.throwProperException) GroupPromoteRequest(org.commonjava.indy.promote.model.GroupPromoteRequest) ApplicationScoped(javax.enterprise.context.ApplicationScoped) StoreType.hosted(org.commonjava.indy.model.core.StoreType.hosted) IndyDataException(org.commonjava.indy.data.IndyDataException) PoolUtils.detectOverloadVoid(org.commonjava.indy.core.ctl.PoolUtils.detectOverloadVoid) AFFECTED_GROUPS(org.commonjava.indy.data.StoreDataManager.AFFECTED_GROUPS) TARGET_STORE(org.commonjava.indy.data.StoreDataManager.TARGET_STORE) ValidationRequest(org.commonjava.indy.promote.validate.model.ValidationRequest) Locker(org.commonjava.cdi.util.weft.Locker) Callable(java.util.concurrent.Callable) Batcher.batch(org.commonjava.indy.promote.util.Batcher.batch) AtomicReference(java.util.concurrent.atomic.AtomicReference) UPLOAD(org.commonjava.maven.galley.model.TransferOperation.UPLOAD) ThreadContext(org.commonjava.cdi.util.weft.ThreadContext) Group(org.commonjava.indy.model.core.Group) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) Transfer(org.commonjava.maven.galley.model.Transfer) Measure(org.commonjava.o11yphant.metrics.annotation.Measure) ExecutorConfig(org.commonjava.cdi.util.weft.ExecutorConfig) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) PromotionValidator(org.commonjava.indy.promote.validate.PromotionValidator) EventUtils.fireEvent(org.commonjava.indy.change.EventUtils.fireEvent) DrainingExecutorCompletionService(org.commonjava.cdi.util.weft.DrainingExecutorCompletionService) DownloadManager(org.commonjava.indy.content.DownloadManager) Event(javax.enterprise.event.Event) PoolUtils.detectOverload(org.commonjava.indy.core.ctl.PoolUtils.detectOverload) PromoteCompleteEvent(org.commonjava.indy.promote.change.event.PromoteCompleteEvent) PromotionHelper.timeInMillSeconds(org.commonjava.indy.promote.data.PromotionHelper.timeInMillSeconds) GroupPromoteResult(org.commonjava.indy.promote.model.GroupPromoteResult) PromotionValidationException(org.commonjava.indy.promote.validate.PromotionValidationException) Logger(org.slf4j.Logger) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) Collections.emptySet(java.util.Collections.emptySet) PromoteConfig(org.commonjava.indy.promote.conf.PromoteConfig) SpecialPathManager(org.commonjava.maven.galley.spi.io.SpecialPathManager) IOException(java.io.IOException) PathsPromoteResult(org.commonjava.indy.promote.model.PathsPromoteResult) ValuePipe(org.commonjava.indy.util.ValuePipe) NotFoundCache(org.commonjava.maven.galley.spi.nfc.NotFoundCache) ExecutionException(java.util.concurrent.ExecutionException) IGNORE_READONLY(org.commonjava.indy.data.StoreDataManager.IGNORE_READONLY) HostedRepository(org.commonjava.indy.model.core.HostedRepository) PromotionHelper.timeInSeconds(org.commonjava.indy.promote.data.PromotionHelper.timeInSeconds) RequestContextHelper(org.commonjava.indy.util.RequestContextHelper) StoreDataManager(org.commonjava.indy.data.StoreDataManager) InputStream(java.io.InputStream) Group(org.commonjava.indy.model.core.Group) Set(java.util.Set) HashSet(java.util.HashSet) Collections.emptySet(java.util.Collections.emptySet) PathsPromoteResult(org.commonjava.indy.promote.model.PathsPromoteResult) ArrayList(java.util.ArrayList) DrainingExecutorCompletionService(org.commonjava.cdi.util.weft.DrainingExecutorCompletionService) IndyWorkflowException(org.commonjava.indy.IndyWorkflowException) StoreDataManager(org.commonjava.indy.data.StoreDataManager) ExecutionException(java.util.concurrent.ExecutionException) HashSet(java.util.HashSet) StoreKey(org.commonjava.indy.model.core.StoreKey) IndyDataException(org.commonjava.indy.data.IndyDataException) PathsPromoteCompleteEvent(org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent) ArtifactStore(org.commonjava.indy.model.core.ArtifactStore) Collection(java.util.Collection)

Example 3 with PathsPromoteCompleteEvent

use of org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent in project indy by Commonjava.

the class EventAuditListener method transformFileGroupingEvent.

private void transformFileGroupingEvent(PromoteCompleteEvent event, FileGroupingEvent fileGroupingEvent) {
    PathsPromoteCompleteEvent pathsPromoteCompleteEvent = (PathsPromoteCompleteEvent) event;
    PathsPromoteResult promoteResult = pathsPromoteCompleteEvent.getPromoteResult();
    String error = promoteResult.getError();
    if (error != null) {
        logger.trace("Error in promoteResult, skip audit.");
        return;
    }
    Set<String> paths = promoteResult.getCompletedPaths();
    if (paths.isEmpty()) {
        logger.trace("No completedPaths, skip audit.");
        return;
    }
    PathsPromoteRequest req = promoteResult.getRequest();
    StoreKey source = req.getSource();
    StoreKey target = req.getTarget();
    // TODO How about generating the key if it does not exist.
    TrackingKey trackingKey = getTrackingKey(source);
    if (trackingKey == null) {
        logger.trace("No tracking key found to: {}", source);
        return;
    }
    Map<String, String> extra = new HashMap<>();
    extra.put(EventConstants.SOURCE, source.toString());
    extra.put(EventConstants.TARGET, target.toString());
    fileGroupingEvent.setExtra(extra);
    fileGroupingEvent.setTimestamp(new Date());
    fileGroupingEvent.setSessionId(trackingKey.getId());
}
Also used : PathsPromoteCompleteEvent(org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent) HashMap(java.util.HashMap) PathsPromoteResult(org.commonjava.indy.promote.model.PathsPromoteResult) PathsPromoteRequest(org.commonjava.indy.promote.model.PathsPromoteRequest) StoreKey(org.commonjava.indy.model.core.StoreKey) Date(java.util.Date) TrackingKey(org.commonjava.indy.folo.model.TrackingKey)

Aggregations

PathsPromoteCompleteEvent (org.commonjava.indy.promote.change.event.PathsPromoteCompleteEvent)3 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections.emptySet (java.util.Collections.emptySet)1 Date (java.util.Date)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 Callable (java.util.concurrent.Callable)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ExecutionException (java.util.concurrent.ExecutionException)1 Future (java.util.concurrent.Future)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 Collectors (java.util.stream.Collectors)1 ApplicationScoped (javax.enterprise.context.ApplicationScoped)1 Event (javax.enterprise.event.Event)1