Examples with Repository com.atlassian.stash.repository.Repository used on opensource projects

Example 16 with Repository

use of com.atlassian.stash.repository.Repository in project stash-codesearch-plugin by palantir.

the class RepositoryServiceManagerImpl method getRepositoryMap.

@Override
public ImmutableMap<String, Repository> getRepositoryMap(PermissionValidationService validationService) {
    PageRequest req = new PageRequestImpl(0, PAGE_SIZE);
    Map<String, Repository> repoMap = new HashMap<String, Repository>();
    while (true) {
        Page<? extends Repository> repoPage = repositoryService.findAll(req);
        for (Repository r : repoPage.getValues()) {
            try {
                if (validationService != null) {
                    validationService.validateForRepository(r, Permission.REPO_READ);
                }
                final String key = r.getProject().getKey() + "^" + r.getSlug();
                if (repoMap.containsKey(key)) {
                    // ITOOLS-13350
                    log.error("Trying to insert existing key '" + key + "' intp repoMap with value " + r.toString());
                    continue;
                }
                repoMap.put(key, r);
            } catch (AuthorisationException e) {
            // User doesn't have permission to access the repo
            }
        }
        if (repoPage.getIsLastPage()) {
            break;
        }
        req = repoPage.getNextPageRequest();
    }
    return ImmutableMap.copyOf(repoMap);
}

Example 17 with Repository

use of com.atlassian.stash.repository.Repository in project stash-codesearch-plugin by palantir.

the class SearchServlet method doGet.

@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    // Make sure user is logged in
    try {
        validationService.validateAuthenticated();
    } catch (AuthorisationException notLoggedInException) {
        try {
            resp.sendRedirect(propertiesService.getLoginUri(URI.create(req.getRequestURL() + (req.getQueryString() == null ? "" : "?" + req.getQueryString()))).toASCIIString());
        } catch (Exception e) {
            log.error("Unable to redirect unauthenticated user to login page", e);
        }
        return;
    }
    // Query and parse settings
    SearchParams params = SearchParams.getParams(req, DateTimeZone.forTimeZone(propertiesService.getDefaultTimeZone()));
    GlobalSettings globalSettings = settingsManager.getGlobalSettings();
    ImmutableSet.Builder<String> noHighlightBuilder = new ImmutableSet.Builder<String>();
    for (String extension : globalSettings.getNoHighlightExtensions().split(",")) {
        extension = extension.trim().toLowerCase();
        if (!extension.isEmpty()) {
            noHighlightBuilder.add(extension);
        }
    }
    ImmutableSet<String> noHighlight = noHighlightBuilder.build();
    int maxPreviewLines = globalSettings.getMaxPreviewLines();
    int maxMatchLines = globalSettings.getMaxMatchLines();
    int maxFragments = globalSettings.getMaxFragments();
    int pageSize = globalSettings.getPageSize();
    TimeValue searchTimeout = new TimeValue(globalSettings.getSearchTimeout());
    float commitHashBoost = (float) globalSettings.getCommitHashBoost();
    float commitSubjectBoost = (float) globalSettings.getCommitBodyBoost();
    float commitBodyBoost = (float) globalSettings.getCommitBodyBoost();
    float fileNameBoost = (float) globalSettings.getFileNameBoost();
    // Execute ES query
    int pages = 0;
    long totalHits = 0;
    long searchTime = 0;
    SearchHit[] currentHits = {};
    String error = "";
    ArrayList<ImmutableMap<String, Object>> hitArray = new ArrayList<ImmutableMap<String, Object>>(currentHits.length);
    ImmutableMap<String, Object> statistics = ImmutableMap.of();
    if (params.doSearch) {
        // Repo map is null iff user is a system administrator (don't need to validate permissions).
        ImmutableMap<String, Repository> repoMap;
        try {
            validationService.validateForGlobal(Permission.SYS_ADMIN);
            repoMap = null;
        } catch (AuthorisationException e) {
            repoMap = repositoryServiceManager.getRepositoryMap(validationService);
            if (repoMap.isEmpty()) {
                error = "You do not have permissions to access any repositories";
            }
        }
        int startIndex = params.page * pageSize;
        SearchRequestBuilder esReq = es.getClient().prepareSearch(ES_SEARCHALIAS).setFrom(startIndex).setSize(pageSize).setTimeout(searchTimeout).setFetchSource(true);
        if (error != null && !error.isEmpty()) {
            log.warn("Not performing search due to error {}", error);
        } else {
            // Build query source and perform query
            QueryBuilder query = matchAllQuery();
            if (params.searchString != null && !params.searchString.isEmpty()) {
                QueryStringQueryBuilder queryStringQuery = queryString(params.searchString).analyzeWildcard(true).lenient(true).defaultOperator(QueryStringQueryBuilder.Operator.AND);
                if (params.searchCommits) {
                    queryStringQuery.field("commit.subject", commitSubjectBoost).field("commit.hash", commitHashBoost).field("commit.body", commitBodyBoost);
                }
                if (params.searchFilenames) {
                    queryStringQuery.field("file.path", fileNameBoost);
                }
                if (params.searchCode) {
                    queryStringQuery.field("file.contents", 1);
                }
                query = queryStringQuery;
            }
            FilterBuilder filter = andFilter(boolFilter().must(repoMap == null ? matchAllFilter() : sf.aclFilter(repoMap), sf.refFilter(params.refNames.split(",")), sf.projectFilter(params.projectKeys.split(",")), sf.repositoryFilter(params.repoNames.split(",")), sf.extensionFilter(params.extensions.split(",")), sf.authorFilter(params.authorNames.split(","))), sf.dateRangeFilter(params.committedAfter, params.committedBefore));
            FilteredQueryBuilder finalQuery = filteredQuery(query, filter);
            esReq.setQuery(finalQuery).setHighlighterPreTags("\u0001").setHighlighterPostTags("\u0001").addHighlightedField("contents", 1, maxFragments);
            String[] typeArray = {};
            if (params.searchCommits) {
                if (params.searchFilenames || params.searchCode) {
                    typeArray = new String[] { "commit", "file" };
                } else {
                    typeArray = new String[] { "commit" };
                }
            } else if (params.searchFilenames || params.searchCode) {
                typeArray = new String[] { "file" };
            }
            esReq.setTypes(typeArray);
            // Build aggregations if statistics were requested
            if (params.showStatistics) {
                esReq.addAggregation(cardinality("authorCardinality").field("authoremail.untouched").precisionThreshold(1000)).addAggregation(terms("authorRanking").field("authoremail.untouched").size(25)).addAggregation(percentiles("charcountPercentiles").field("charcount").percentiles(PERCENTILES)).addAggregation(extendedStats("charcountStats").field("charcount")).addAggregation(filter("commitCount").filter(typeFilter("commit"))).addAggregation(cardinality("extensionCardinality").field("extension").precisionThreshold(1000)).addAggregation(terms("extensionRanking").field("extension").size(25)).addAggregation(percentiles("linecountPercentiles").field("linecount").percentiles(PERCENTILES)).addAggregation(extendedStats("linecountStats").field("linecount"));
            }
            SearchResponse esResp = null;
            try {
                esResp = esReq.get();
            } catch (SearchPhaseExecutionException e) {
                log.warn("Query failure", e);
                error = "Make sure your query conforms to the Lucene/Elasticsearch query string syntax.";
            }
            if (esResp != null) {
                SearchHits esHits = esResp.getHits();
                totalHits = esHits.getTotalHits();
                pages = (int) Math.min(Integer.MAX_VALUE, (totalHits + pageSize - 1) / pageSize);
                currentHits = esHits.getHits();
                searchTime = esResp.getTookInMillis();
                for (ShardSearchFailure failure : esResp.getShardFailures()) {
                    log.warn("Shard failure {}", failure.reason());
                    if (error == null || error.isEmpty()) {
                        error = "Shard failure: " + failure.reason();
                    }
                }
                Aggregations aggs = esResp.getAggregations();
                if (params.showStatistics && aggs != null && !aggs.asList().isEmpty()) {
                    Cardinality authorCardinality = aggs.get("authorCardinality");
                    Terms authorRanking = aggs.get("authorRanking");
                    Percentiles charcountPercentiles = aggs.get("charcountPercentiles");
                    Filter commitCount = aggs.get("commitCount");
                    ExtendedStats charcountStats = aggs.get("charcountStats");
                    Cardinality extensionCardinality = aggs.get("extensionCardinality");
                    Terms extensionRanking = aggs.get("extensionRanking");
                    Percentiles linecountPercentiles = aggs.get("linecountPercentiles");
                    ExtendedStats linecountStats = aggs.get("linecountStats");
                    statistics = new ImmutableMap.Builder<String, Object>().put("authorCardinality", authorCardinality.getValue()).put("authorRanking", getSoyRankingList(authorRanking, commitCount.getDocCount())).put("charcount", new ImmutableMap.Builder<String, Object>().put("average", charcountStats.getAvg()).put("max", Math.round(charcountStats.getMax())).put("min", Math.round(charcountStats.getMin())).put("percentiles", getSoyPercentileList(charcountPercentiles, PERCENTILES)).put("sum", Math.round(charcountStats.getSum())).build()).put("commitcount", commitCount.getDocCount()).put("extensionCardinality", extensionCardinality.getValue()).put("extensionRanking", getSoyRankingList(extensionRanking, charcountStats.getCount())).put("filecount", charcountStats.getCount()).put("linecount", new ImmutableMap.Builder<String, Object>().put("average", linecountStats.getAvg()).put("max", Math.round(linecountStats.getMax())).put("min", Math.round(linecountStats.getMin())).put("percentiles", getSoyPercentileList(linecountPercentiles, PERCENTILES)).put("sum", Math.round(linecountStats.getSum())).build()).build();
                }
            }
        }
        // Iterate through current page of search hits
        for (SearchHit hit : currentHits) {
            ImmutableMap<String, Object> hitData = searchHitToDataMap(hit, repoMap, maxPreviewLines, maxMatchLines, noHighlight);
            if (hitData != null) {
                hitArray.add(hitData);
            }
        }
    }
    // Render page
    pbs.assembler().resources().requireContext("com.atlassian.auiplugin:aui-date-picker");
    pbs.assembler().resources().requireContext("com.atlassian.auiplugin:aui-experimental-tooltips");
    pbs.assembler().resources().requireWebResource("com.palantir.stash.stash-code-search:scs-resources");
    resp.setContentType("text/html");
    try {
        String queryString = req.getQueryString();
        String fullUri = req.getRequestURI() + "?" + (queryString == null ? "" : queryString.replaceAll("&?page=\\d*", ""));
        ImmutableMap<String, Object> data = new ImmutableMap.Builder<String, Object>().put("pages", pages).put("currentPage", params.page).put("prevParams", params.soyParams).put("doSearch", params.doSearch).put("totalHits", totalHits).put("hitArray", hitArray).put("statistics", statistics).put("error", error).put("fullUri", fullUri).put("baseUrl", propertiesService.getBaseUrl().toASCIIString()).put("resultFrom", Math.min(totalHits, params.page * pageSize + 1)).put("resultTo", Math.min(totalHits, (params.page + 1) * pageSize)).put("searchTime", searchTime).build();
        soyTemplateRenderer.render(resp.getWriter(), "com.palantir.stash.stash-code-search:codesearch-soy", "plugin.page.codesearch.searchPage", data);
    } catch (Exception e) {
        log.error("Error rendering Soy template", e);
    }
}

Example 18 with Repository

use of com.atlassian.stash.repository.Repository in project stash-codesearch-plugin by palantir.

the class SearchServlet method searchHitToDataMap.

// Returns map view of search hits for soy templates
// Not sure this is actually safe at all.  Jerry thought so.
@SuppressWarnings("unchecked")
private ImmutableMap<String, Object> searchHitToDataMap(SearchHit hit, // null iff no permission validation required
Map<String, Repository> repoMap, int maxPreviewLines, int maxMatchLines, ImmutableSet<String> noHighlight) {
    ImmutableMap.Builder<String, Object> hitData = new ImmutableMap.Builder<String, Object>();
    Map<String, Object> hitSource = hit.getSource();
    String type = hit.getType();
    hitData.put("type", type);
    String project = getStringFromMap(hitSource, "project");
    hitData.put("project", project);
    String repository = getStringFromMap(hitSource, "repository");
    hitData.put("repository", repository);
    // Validate permissions & build hit data map
    String repoId = project + "^" + repository;
    Repository repoObject;
    if (repoMap == null) {
        // current user is system administrator
        repoObject = repositoryServiceManager.getRepositoryService().getBySlug(project, repository);
    } else {
        // must validate against allowed repositories for non-administrators
        repoObject = repoMap.get(repoId);
    }
    if (repoObject != null && repoObject.getProject().getKey().equals(project) && repoObject.getSlug().equals(repository)) {
        // Generate refs array
        ImmutableSortedSet<String> refSet;
        try {
            refSet = ImmutableSortedSet.copyOf((Iterable<String>) hitSource.get("refs"));
        } catch (Exception e) {
            log.warn("Invalid refs collection detected for element in {}/{}", project, repository, e);
            return null;
        }
        if (refSet.isEmpty()) {
            log.warn("Detected empty refs collection for element in {}/{}", project, repository);
            return null;
        }
        hitData.put("refs", refSet);
        // Human-readable labels
        hitData.put("projectname", repoObject.getProject().getName()).put("repositoryname", repoObject.getName());
        if (type.equals("commit")) {
            hitData.put("hash", getStringFromMap(hitSource, "hash")).put("subject", getStringFromMap(hitSource, "subject")).put("body", getStringFromMap(hitSource, "body")).put("commitDate", getDateStringFromMap(hitSource, "commitdate")).put("authorName", getStringFromMap(hitSource, "authorname")).put("authorEmail", getStringFromMap(hitSource, "authoremail"));
        } else if (type.equals("file")) {
            HighlightField highlightField = hit.getHighlightFields().get("contents");
            String path = getStringFromMap(hitSource, "path");
            String primaryRef = "refs/heads/master";
            if (!refSet.contains(primaryRef)) {
                primaryRef = refSet.iterator().next();
            }
            String contents = getStringFromMap(hitSource, "contents");
            SourceSearch searchedContents = SourceSearch.search(contents, highlightField, 1, maxPreviewLines, maxMatchLines);
            String extension = getStringFromMap(hitSource, "extension");
            hitData.put("path", path).put("blob", getStringFromMap(hitSource, "blob")).put("primaryRef", primaryRef).put("sourceLines", searchedContents.getJoinedLines()).put("sourceLineNums", searchedContents.getJoinedLineNums()).put("isPreview", searchedContents.isPreview()).put("shownLines", searchedContents.getLines().length).put("excessLines", searchedContents.getExcess()).put("extension", extension).put("noHighlight", noHighlight.contains(extension));
        }
    } else {
        return null;
    }
    return hitData.build();
}

Example 19 with Repository

use of com.atlassian.stash.repository.Repository in project stash-codesearch-plugin by palantir.

the class RepositorySettingsServlet method doPost.

@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    if (!verifyLoggedIn(req, resp)) {
        return;
    }
    final Repository repository = getRepository(req);
    if (repository == null) {
        resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Repo not found.");
        return;
    }
    if (!verifyRepoAdmin(req, resp, repository)) {
        return;
    }
    // Parse arguments
    ArrayList<String> errors = new ArrayList<String>();
    String refRegex = req.getParameter("refRegex");
    try {
        Pattern.compile(refRegex);
    } catch (Exception e) {
        errors.add("Invalid regex: \"" + refRegex + "\"");
    }
    // Update settings object iff no parse errors
    RepositorySettings settings;
    if (errors.isEmpty()) {
        settings = settingsManager.setRepositorySettings(repository, refRegex);
        if ("true".equals(req.getParameter("reindex"))) {
            // Reindex is requested
            log.info("User {} submitted an async reindex for {}^{}", req.getRemoteUser(), repository.getProject().getKey(), repository.getSlug());
            new Thread(new Runnable() {

                @Override
                public void run() {
                    try {
                        /* XXX: Is this really necessary?  Users of the
                             * configuration page are already repo admins, I
                             * think, and it doesn't look like reindexing does
                             * anything that needs this permission (although
                             * maybe you need to have permission to see all
                             * branches, which  a repo admin has?)
                             */
                        EscalatedSecurityContext esc = securityService.withPermission(Permission.REPO_ADMIN, "reindex by repo admin");
                        esc.call(new Operation<Void, Exception>() {

                            @Override
                            public Void perform() {
                                searchUpdater.reindexRepository(repository.getProject().getKey(), repository.getSlug());
                                return null;
                            }
                        });
                    } catch (Exception e) {
                        log.warn("Caught exception while reindexing", e);
                    }
                }
            }).start();
        }
    } else {
        settings = settingsManager.getRepositorySettings(repository);
    }
    renderPage(req, resp, repository, settings, errors);
}

Example 20 with Repository

use of com.atlassian.stash.repository.Repository in project stash-codesearch-plugin by palantir.

the class SearchFilterUtils method aclFilter.

public FilterBuilder aclFilter(Map<String, Repository> repoMap) {
    if (repoMap.isEmpty()) {
        return boolFilter().mustNot(matchAllFilter());
    }
    // Compute cryptographic hash of repository set to use for cache key
    String[] projectRepoPairs = repoMap.keySet().toArray(new String[repoMap.size()]);
    Arrays.sort(projectRepoPairs);
    String filterHash;
    try {
        MessageDigest hasher = MessageDigest.getInstance("SHA-256");
        for (String pair : projectRepoPairs) {
            hasher.update(pair.getBytes());
            hasher.update((byte) 0);
        }
        filterHash = new String(Base64.encodeBase64(hasher.digest()));
    } catch (Exception e) {
        filterHash = null;
        log.error("Caught exception generating ACL hash -- caching is disabled.", e);
    }
    // Create disjunction of individual repo ACL filters
    BoolFilterBuilder filter = boolFilter();
    if (filterHash != null) {
        filter.cache(true).cacheKey("CACHE^ACLORFILTER^" + filterHash);
    } else {
        filter.cache(false);
    }
    for (Repository repo : repoMap.values()) {
        filter.should(projectRepositoryFilter(repo.getProject().getKey(), repo.getSlug()));
    }
    return filter;
}