Search in sources :

Example 1 with TemporaryGroupIndex

use of org.apache.archiva.indexer.merger.TemporaryGroupIndex in project archiva by apache.

the class DefaultIndexMerger method buildMergedIndex.

@Override
public IndexingContext buildMergedIndex(IndexMergerRequest indexMergerRequest) throws IndexMergerException {
    String groupId = indexMergerRequest.getGroupId();
    if (runningGroups.contains(groupId)) {
        log.info("skip build merge remote indexes for id: '{}' as already running", groupId);
        return null;
    }
    runningGroups.add(groupId);
    StopWatch stopWatch = new StopWatch();
    stopWatch.reset();
    stopWatch.start();
    Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory();
    String tempRepoId = mergedIndexDirectory.getFileName().toString();
    try {
        Path indexLocation = mergedIndexDirectory.resolve(indexMergerRequest.getMergedIndexPath());
        List<IndexingContext> members = indexMergerRequest.getRepositoriesIds().stream().map(id -> repositoryRegistry.getRepository(id)).filter(repo -> repo.getType().equals(RepositoryType.MAVEN)).map(repo -> {
            try {
                return repo.getIndexingContext().getBaseContext(IndexingContext.class);
            } catch (UnsupportedBaseContextException e) {
                return null;
            // Ignore
            }
        }).filter(Objects::nonNull).collect(Collectors.toList());
        ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
        IndexingContext mergedCtx = indexer.createMergedIndexingContext(tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), indexLocation.toFile(), true, memberProvider);
        mergedCtx.optimize();
        if (indexMergerRequest.isPackIndex()) {
            IndexPackingRequest request = new // 
            IndexPackingRequest(// 
            mergedCtx, // 
            mergedCtx.acquireIndexSearcher().getIndexReader(), indexLocation.toFile());
            indexPacker.packIndex(request);
        }
        if (indexMergerRequest.isTemporary()) {
            temporaryGroupIndexes.add(new TemporaryGroupIndex(mergedIndexDirectory, tempRepoId, groupId, indexMergerRequest.getMergedIndexTtl()));
            temporaryContextes.add(mergedCtx);
        }
        stopWatch.stop();
        log.info("merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(), stopWatch.getTime());
        return mergedCtx;
    } catch (IOException e) {
        throw new IndexMergerException(e.getMessage(), e);
    } finally {
        runningGroups.remove(groupId);
    }
}
Also used : Path(java.nio.file.Path) Async(org.springframework.scheduling.annotation.Async) UnsupportedBaseContextException(org.apache.archiva.indexer.UnsupportedBaseContextException) StaticContextMemberProvider(org.apache.maven.index.context.StaticContextMemberProvider) LoggerFactory(org.slf4j.LoggerFactory) IndexCreator(org.apache.maven.index.context.IndexCreator) Inject(javax.inject.Inject) Service(org.springframework.stereotype.Service) FileUtils(org.apache.archiva.common.utils.FileUtils) IndexPackingRequest(org.apache.maven.index.packer.IndexPackingRequest) Path(java.nio.file.Path) Logger(org.slf4j.Logger) IndexPacker(org.apache.maven.index.packer.IndexPacker) Files(java.nio.file.Files) ContextMemberProvider(org.apache.maven.index.context.ContextMemberProvider) Collection(java.util.Collection) RepositoryRegistry(org.apache.archiva.repository.RepositoryRegistry) IOException(java.io.IOException) TemporaryGroupIndex(org.apache.archiva.indexer.merger.TemporaryGroupIndex) RepositoryType(org.apache.archiva.repository.RepositoryType) Collectors(java.util.stream.Collectors) IndexMerger(org.apache.archiva.indexer.merger.IndexMerger) IndexMergerException(org.apache.archiva.indexer.merger.IndexMergerException) Indexer(org.apache.maven.index.Indexer) Objects(java.util.Objects) List(java.util.List) IndexingContext(org.apache.maven.index.context.IndexingContext) IndexMergerRequest(org.apache.archiva.indexer.merger.IndexMergerRequest) StopWatch(org.apache.commons.lang.time.StopWatch) Optional(java.util.Optional) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) TemporaryGroupIndex(org.apache.archiva.indexer.merger.TemporaryGroupIndex) StaticContextMemberProvider(org.apache.maven.index.context.StaticContextMemberProvider) ContextMemberProvider(org.apache.maven.index.context.ContextMemberProvider) IOException(java.io.IOException) StopWatch(org.apache.commons.lang.time.StopWatch) UnsupportedBaseContextException(org.apache.archiva.indexer.UnsupportedBaseContextException) IndexPackingRequest(org.apache.maven.index.packer.IndexPackingRequest) IndexingContext(org.apache.maven.index.context.IndexingContext) StaticContextMemberProvider(org.apache.maven.index.context.StaticContextMemberProvider) IndexMergerException(org.apache.archiva.indexer.merger.IndexMergerException)

Example 2 with TemporaryGroupIndex

use of org.apache.archiva.indexer.merger.TemporaryGroupIndex in project archiva by apache.

the class TemporaryGroupIndexSessionCleaner method sessionDestroyed.

@Override
public void sessionDestroyed(HttpSessionEvent httpSessionEvent) {
    @SuppressWarnings("unchecked") Map<String, TemporaryGroupIndex> tempFilesPerKey = (Map<String, TemporaryGroupIndex>) httpSessionEvent.getSession().getAttribute(TEMPORARY_INDEX_SESSION_KEY);
    for (TemporaryGroupIndex temporaryGroupIndex : tempFilesPerKey.values()) {
        log.info("cleanup temporaryGroupIndex {} directory {}", temporaryGroupIndex.getIndexId(), temporaryGroupIndex.getDirectory().toAbsolutePath());
        getIndexMerger(httpSessionEvent).cleanTemporaryGroupIndex(temporaryGroupIndex);
    }
}
Also used : TemporaryGroupIndex(org.apache.archiva.indexer.merger.TemporaryGroupIndex) Map(java.util.Map) HashMap(java.util.HashMap)

Example 3 with TemporaryGroupIndex

use of org.apache.archiva.indexer.merger.TemporaryGroupIndex in project archiva by apache.

the class ArchivaDavResourceFactory method buildMergedIndexDirectory.

protected Path buildMergedIndexDirectory(List<String> repositories, String activePrincipal, DavServletRequest request, RepositoryGroupConfiguration repositoryGroupConfiguration) throws DavException {
    try {
        HttpSession session = request.getSession();
        @SuppressWarnings("unchecked") Map<String, TemporaryGroupIndex> temporaryGroupIndexMap = (Map<String, TemporaryGroupIndex>) session.getAttribute(TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY);
        if (temporaryGroupIndexMap == null) {
            temporaryGroupIndexMap = new HashMap<>();
        }
        TemporaryGroupIndex tmp = temporaryGroupIndexMap.get(repositoryGroupConfiguration.getId());
        if (tmp != null && tmp.getDirectory() != null && Files.exists(tmp.getDirectory())) {
            if (System.currentTimeMillis() - tmp.getCreationTime() > (repositoryGroupConfiguration.getMergedIndexTtl() * 60 * 1000)) {
                log.debug(MarkerFactory.getMarker("group.merged.index"), "tmp group index '{}' is too old so delete it", repositoryGroupConfiguration.getId());
                indexMerger.cleanTemporaryGroupIndex(tmp);
            } else {
                log.debug(MarkerFactory.getMarker("group.merged.index"), "merged index for group '{}' found in cache", repositoryGroupConfiguration.getId());
                return tmp.getDirectory();
            }
        }
        Set<String> authzRepos = new HashSet<String>();
        String permission = WebdavMethodUtil.getMethodPermission(request.getMethod());
        for (String repository : repositories) {
            try {
                if (servletAuth.isAuthorized(activePrincipal, repository, permission)) {
                    authzRepos.add(repository);
                    authzRepos.addAll(this.repositorySearch.getRemoteIndexingContextIds(repository));
                }
            } catch (UnauthorizedException e) {
                // TODO: review exception handling
                log.debug("Skipping repository '{}' for user '{}': {}", repository, activePrincipal, e.getMessage());
            }
        }
        log.info("generate temporary merged index for repository group '{}' for repositories '{}'", repositoryGroupConfiguration.getId(), authzRepos);
        Path tempRepoFile = Files.createTempDirectory("temp");
        tempRepoFile.toFile().deleteOnExit();
        IndexMergerRequest indexMergerRequest = new IndexMergerRequest(authzRepos, true, repositoryGroupConfiguration.getId(), repositoryGroupConfiguration.getMergedIndexPath(), repositoryGroupConfiguration.getMergedIndexTtl()).mergedIndexDirectory(tempRepoFile).temporary(true);
        MergedRemoteIndexesTaskRequest taskRequest = new MergedRemoteIndexesTaskRequest(indexMergerRequest, indexMerger);
        MergedRemoteIndexesTask job = new MergedRemoteIndexesTask(taskRequest);
        IndexingContext indexingContext = job.execute().getIndexingContext();
        Path mergedRepoDir = indexingContext.getIndexDirectoryFile().toPath();
        TemporaryGroupIndex temporaryGroupIndex = new TemporaryGroupIndex(mergedRepoDir, indexingContext.getId(), repositoryGroupConfiguration.getId(), // 
        repositoryGroupConfiguration.getMergedIndexTtl()).setCreationTime(new Date().getTime());
        temporaryGroupIndexMap.put(repositoryGroupConfiguration.getId(), temporaryGroupIndex);
        session.setAttribute(TemporaryGroupIndexSessionCleaner.TEMPORARY_INDEX_SESSION_KEY, temporaryGroupIndexMap);
        return mergedRepoDir;
    } catch (RepositorySearchException e) {
        throw new DavException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
    } catch (IndexMergerException e) {
        throw new DavException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
    } catch (IOException e) {
        throw new DavException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
    }
}
Also used : Path(java.nio.file.Path) TemporaryGroupIndex(org.apache.archiva.indexer.merger.TemporaryGroupIndex) MergedRemoteIndexesTaskRequest(org.apache.archiva.indexer.merger.MergedRemoteIndexesTaskRequest) MergedRemoteIndexesTask(org.apache.archiva.indexer.merger.MergedRemoteIndexesTask) DavException(org.apache.jackrabbit.webdav.DavException) HttpSession(javax.servlet.http.HttpSession) IndexMergerRequest(org.apache.archiva.indexer.merger.IndexMergerRequest) RepositorySearchException(org.apache.archiva.indexer.search.RepositorySearchException) IOException(java.io.IOException) Date(java.util.Date) UnauthorizedException(org.apache.archiva.redback.authorization.UnauthorizedException) IndexingContext(org.apache.maven.index.context.IndexingContext) IndexMergerException(org.apache.archiva.indexer.merger.IndexMergerException) Map(java.util.Map) HashMap(java.util.HashMap) HashSet(java.util.HashSet)

Example 4 with TemporaryGroupIndex

use of org.apache.archiva.indexer.merger.TemporaryGroupIndex in project archiva by apache.

the class DefaultIndexMerger method cleanTemporaryGroupIndex.

@Async
@Override
public void cleanTemporaryGroupIndex(TemporaryGroupIndex temporaryGroupIndex) {
    if (temporaryGroupIndex == null) {
        return;
    }
    try {
        Optional<IndexingContext> ctxOpt = temporaryContextes.stream().filter(ctx -> ctx.getId().equals(temporaryGroupIndex.getIndexId())).findFirst();
        if (ctxOpt.isPresent()) {
            IndexingContext ctx = ctxOpt.get();
            indexer.closeIndexingContext(ctx, true);
            temporaryGroupIndexes.remove(temporaryGroupIndex);
            temporaryContextes.remove(ctx);
            Path directory = temporaryGroupIndex.getDirectory();
            if (directory != null && Files.exists(directory)) {
                FileUtils.deleteDirectory(directory);
            }
        }
    } catch (IOException e) {
        log.warn("fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e);
    }
}
Also used : Async(org.springframework.scheduling.annotation.Async) UnsupportedBaseContextException(org.apache.archiva.indexer.UnsupportedBaseContextException) StaticContextMemberProvider(org.apache.maven.index.context.StaticContextMemberProvider) LoggerFactory(org.slf4j.LoggerFactory) IndexCreator(org.apache.maven.index.context.IndexCreator) Inject(javax.inject.Inject) Service(org.springframework.stereotype.Service) FileUtils(org.apache.archiva.common.utils.FileUtils) IndexPackingRequest(org.apache.maven.index.packer.IndexPackingRequest) Path(java.nio.file.Path) Logger(org.slf4j.Logger) IndexPacker(org.apache.maven.index.packer.IndexPacker) Files(java.nio.file.Files) ContextMemberProvider(org.apache.maven.index.context.ContextMemberProvider) Collection(java.util.Collection) RepositoryRegistry(org.apache.archiva.repository.RepositoryRegistry) IOException(java.io.IOException) TemporaryGroupIndex(org.apache.archiva.indexer.merger.TemporaryGroupIndex) RepositoryType(org.apache.archiva.repository.RepositoryType) Collectors(java.util.stream.Collectors) IndexMerger(org.apache.archiva.indexer.merger.IndexMerger) IndexMergerException(org.apache.archiva.indexer.merger.IndexMergerException) Indexer(org.apache.maven.index.Indexer) Objects(java.util.Objects) List(java.util.List) IndexingContext(org.apache.maven.index.context.IndexingContext) IndexMergerRequest(org.apache.archiva.indexer.merger.IndexMergerRequest) StopWatch(org.apache.commons.lang.time.StopWatch) Optional(java.util.Optional) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) Path(java.nio.file.Path) IndexingContext(org.apache.maven.index.context.IndexingContext) IOException(java.io.IOException) Async(org.springframework.scheduling.annotation.Async)

Aggregations

TemporaryGroupIndex (org.apache.archiva.indexer.merger.TemporaryGroupIndex)4 IOException (java.io.IOException)3 Path (java.nio.file.Path)3 IndexMergerException (org.apache.archiva.indexer.merger.IndexMergerException)3 IndexMergerRequest (org.apache.archiva.indexer.merger.IndexMergerRequest)3 IndexingContext (org.apache.maven.index.context.IndexingContext)3 Files (java.nio.file.Files)2 Collection (java.util.Collection)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Objects (java.util.Objects)2 Optional (java.util.Optional)2 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)2 Collectors (java.util.stream.Collectors)2 Inject (javax.inject.Inject)2 FileUtils (org.apache.archiva.common.utils.FileUtils)2 UnsupportedBaseContextException (org.apache.archiva.indexer.UnsupportedBaseContextException)2 IndexMerger (org.apache.archiva.indexer.merger.IndexMerger)2 RepositoryRegistry (org.apache.archiva.repository.RepositoryRegistry)2