Examples with ArtifactStore org.commonjava.indy.model.core.ArtifactStore used on opensource projects

Example 31 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class DownloadManagerTest method downloadOnePOMFromSecondRepositoryAfterDummyRepoFails.

@Test
public void downloadOnePOMFromSecondRepositoryAfterDummyRepoFails() throws Exception {
    final RemoteRepository repo = new RemoteRepository(MAVEN_PKG_KEY, "dummy", "http://www.nowhere.com/");
    final String content = "This is a test";
    final String path = "/org/apache/maven/maven-model/3.0.3/maven-model-3.0.3.pom";
    final RemoteRepository repo2 = new RemoteRepository(MAVEN_PKG_KEY, "central", "http://repo.maven.apache.org/maven2");
    fixture.getTransport().registerDownload(new ConcreteResource(new RepositoryLocation(repo2), path), new TestDownload(content.getBytes()));
    data.storeArtifactStore(repo, summary, false, true, new EventMetadata());
    data.storeArtifactStore(repo2, summary, false, true, new EventMetadata());
    final List<ArtifactStore> repos = new ArrayList<ArtifactStore>();
    repos.add(repo);
    repos.add(repo2);
    final Transfer stream = downloader.retrieveFirst(repos, path, new EventMetadata());
    final String downloaded = IOUtils.toString(stream.openInputStream());
    assertThat(downloaded, equalTo(content));
}

Example 32 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class PromotionValidationTools method addLocations.

public void addLocations(final List<Location> locations, final StoreKey... extraLocations) throws IndyDataException {
    for (StoreKey extra : extraLocations) {
        ArtifactStore store = getArtifactStore(extra);
        locations.add(LocationUtils.toLocation(store));
    }
}

Example 33 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class PromotionManager method runPathPromotions.

private PathsPromoteResult runPathPromotions(final PathsPromoteRequest request, final Set<String> pending, final List<Transfer> contents, final ValidationResult validation) {
    long begin = System.currentTimeMillis();
    PromotionHelper.PromotionRepoRetrievalResult checkResult = promotionHelper.checkAndRetrieveSourceAndTargetRepos(request);
    if (checkResult.hasErrors()) {
        return new PathsPromoteResult(request, pending, emptySet(), emptySet(), StringUtils.join(checkResult.errors, "\n"), validation);
    }
    final ArtifactStore targetStore = checkResult.targetStore;
    StoreKey targetKey = targetStore.getKey();
    logger.info("Run promotion from: {} to: {}, paths: {}", request.getSource(), targetKey, pending);
    Set<Group> affectedGroups;
    try {
        affectedGroups = storeManager.query().getGroupsAffectedBy(targetKey);
        logger.info("Calculate affected groups, target: {}, affected-groups: {}", targetKey, affectedGroups);
    } catch (IndyDataException e) {
        logger.error("Get affected groups failed", e);
        return new PathsPromoteResult(request, pending, emptySet(), emptySet(), "Get affected groups failed, " + e.getMessage(), validation);
    }
    DrainingExecutorCompletionService<Set<PathTransferResult>> svc = new DrainingExecutorCompletionService<>(transferService);
    int corePoolSize = transferService.getCorePoolSize();
    int size = contents.size();
    int batchSize = getParalleledBatchSize(size, corePoolSize);
    logger.info("Execute parallel on collection, size: {}, batch: {}", size, batchSize);
    Collection<Collection<Transfer>> batches = batch(contents, batchSize);
    final List<String> errors = new ArrayList<>();
    try {
        detectOverloadVoid(() -> batches.forEach(batch -> svc.submit(newPathPromotionsJob(batch, targetStore, request, affectedGroups))));
    } catch (IndyWorkflowException e) {
        // might be PoolOverloadException. Log it and continue to revert any completed paths
        String msg = String.format("Failed to submit all path promotion jobs. Error: %s", e.toString());
        logger.error(msg, e);
        errors.add(msg);
    }
    final Set<PathTransferResult> results = new HashSet<>();
    try {
        svc.drain(results::addAll);
    } catch (InterruptedException | ExecutionException e) {
        String msg = String.format("Error waiting for promotion of: %s to: %s", request.getSource(), request.getTarget());
        logger.error(msg, e);
        errors.add(msg);
    }
    final Set<String> completed = new HashSet<>();
    final Set<String> skipped = new HashSet<>();
    results.forEach(result -> {
        if (result.error != null) {
            errors.add(result.error);
        } else if (result.skipped) {
            skipped.add(result.path);
        } else {
            completed.add(result.path);
        }
    });
    PathsPromoteResult result;
    if (!errors.isEmpty()) {
        List<String> rollbackErrors = promotionHelper.deleteFromStore(completed, targetStore);
        errors.addAll(rollbackErrors);
        result = new PathsPromoteResult(request, pending, emptySet(), emptySet(), StringUtils.join(errors, "\n"), validation);
    } else {
        result = new PathsPromoteResult(request, emptySet(), completed, skipped, null, validation);
        final String name = String.format("PromoteNFCClean-method(%s)-source(%s)-target(%s)", "runPathPromotions", request.getSource(), targetStore.getKey());
        final String context = String.format("Class: %s, method: %s, source: %s, target: %s", this.getClass().getName(), "runPathPromotions", request.getSource(), targetStore.getKey());
        storeManager.asyncGroupAffectedBy(new StoreDataManager.ContextualTask(name, context, () -> promotionHelper.clearStoreNFC(completed, targetStore, affectedGroups)));
        if (request.isFireEvents()) {
            fireEvent(promoteCompleteEvent, new PathsPromoteCompleteEvent(result));
        }
    }
    logger.info("Promotion completed, promotionId: {}, timeInSeconds: {}", request.getPromotionId(), timeInSeconds(begin));
    return result;
}

Example 34 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class PromotionValidator method validate.

/**
 * NOTE: As of Indy 1.2.6, ValidationRequest passed back to enable further post-processing, especially of promotion
 * paths, after promotion takes place. This enables us to avoid re-executing recursive path discovery, for instance.
 *
 * @param request
 * @param result
 * @param baseUrl
 * @return
 * @throws PromotionValidationException
 * @throws IndyWorkflowException
 */
@Measure
public ValidationRequest validate(PromoteRequest request, ValidationResult result, String baseUrl) throws PromotionValidationException, IndyWorkflowException {
    ValidationRuleSet set = validationsManager.getRuleSetMatching(request.getTargetKey());
    ArtifactStore source;
    try {
        source = storeDataMgr.getArtifactStore(request.getSource());
    } catch (IndyDataException e) {
        throw new PromotionValidationException(String.format("Failed to retrieve source ArtifactStore: %s for validation", request.getSource()), e);
    }
    if (set != null) {
        result.setRuleSet(set.getName());
        RequestContextHelper.setContext(PROMOTION_VALIDATION_RULE_SET, set.getName());
        logger.debug("Running validation rule-set for promotion: {}", set.getName());
        List<String> ruleNames = set.getRuleNames();
        if (ruleNames != null && !ruleNames.isEmpty()) {
            final ArtifactStore store = getRequestStore(request, baseUrl);
            final ValidationRequest req = new ValidationRequest(request, set, validationTools, store);
            try {
                DrainingExecutorCompletionService<Exception> svc = new DrainingExecutorCompletionService<>(validateService);
                detectOverloadVoid(() -> {
                    for (String ruleRef : ruleNames) {
                        svc.submit(() -> {
                            RequestContextHelper.setContext(PROMOTION_VALIDATION_RULE, ruleRef);
                            Exception err = null;
                            try {
                                executeValidationRule(ruleRef, req, result, request);
                            } catch (Exception e) {
                                err = e;
                            } finally {
                                RequestContextHelper.clearContext(PROMOTION_VALIDATION_RULE);
                            }
                            return err;
                        });
                    }
                });
                List<String> errors = new ArrayList<>();
                svc.drain(err -> {
                    if (err != null) {
                        logger.error("Promotion validation failure", err);
                        errors.add(err.getMessage());
                    }
                });
                if (!errors.isEmpty()) {
                    throw new PromotionValidationException(format("Failed to do promotion validation: \n\n%s", join(errors, "\n")));
                }
            } catch (InterruptedException e) {
                throw new PromotionValidationException("Failed to do promotion validation: validation execution has been interrupted ", e);
            } catch (ExecutionException e) {
                throw new PromotionValidationException("Failed to execute promotion validations", e);
            } finally {
                if (needTempRepo(request)) {
                    try {
                        final String changeSum = format("Removes the temp remote repo [%s] after promote operation.", store);
                        storeDataMgr.deleteArtifactStore(store.getKey(), new ChangeSummary(ChangeSummary.SYSTEM_USER, changeSum), new EventMetadata().set(ContentManager.SUPPRESS_EVENTS, true));
                        Transfer root = downloadManager.getStoreRootDirectory(store);
                        if (root.exists()) {
                            root.delete(false);
                        }
                        logger.debug("Promotion temporary repo {} has been deleted for {}", store.getKey(), request.getSource());
                    } catch (IndyDataException | IOException e) {
                        logger.warn("Temporary promotion validation repository was NOT removed correctly.", e);
                    }
                }
            }
            return req;
        } else {
            logger.info("No validation rules are defined for: {}", request.getTargetKey());
            return new ValidationRequest(request, set, validationTools, source);
        }
    } else {
        logger.info("No validation rule-sets are defined for: {}", request.getTargetKey());
        return new ValidationRequest(request, set, validationTools, source);
    }
}

Example 35 with ArtifactStore

use of org.commonjava.indy.model.core.ArtifactStore in project indy by Commonjava.

the class PromotionHelper method checkAndRetrieveSourceAndTargetRepos.

/**
 * Check whether the source and target repo exists.
 * @param request
 * @return errors
 */
PromotionRepoRetrievalResult checkAndRetrieveSourceAndTargetRepos(PathsPromoteRequest request) {
    List<String> errors = new ArrayList<>();
    ArtifactStore sourceStore = null;
    ArtifactStore targetStore = null;
    try {
        sourceStore = storeManager.getArtifactStore(request.getSource());
    } catch (IndyDataException e) {
        String msg = String.format("Failed to retrieve source store: %s. Reason: %s", request.getSource(), e.getMessage());
        logger.error(msg, e);
        errors.add(msg);
    }
    try {
        targetStore = storeManager.getArtifactStore(request.getTarget());
    } catch (IndyDataException e) {
        String msg = String.format("Failed to retrieve target store: %s. Reason: %s", request.getTarget(), e.getMessage());
        logger.error(msg, e);
        errors.add(msg);
    }
    if (targetStore == null || sourceStore == null) {
        String msg = String.format("Failed to retrieve stores, source: %s, target: %s", request.getSource(), request.getTarget());
        logger.error(msg);
        errors.add(msg);
    }
    return new PromotionRepoRetrievalResult(errors, sourceStore, targetStore);
}