Examples with CodeSearcher com.searchcode.app.service.CodeSearcher used on opensource projects

Example 1 with CodeSearcher

use of com.searchcode.app.service.CodeSearcher in project searchcode-server by boyter.

the class IndexBaseAndGitRepoJobTest method testMissingPathFilesShouldPage.

public void testMissingPathFilesShouldPage() {
    IndexGitRepoJob gitRepoJob = new IndexGitRepoJob();
    CodeSearcher codeSearcherMock = Mockito.mock(CodeSearcher.class);
    List<String> repoReturn = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        repoReturn.add("string" + i);
    }
    when(codeSearcherMock.getRepoDocuments("testRepoName", 0)).thenReturn(repoReturn);
    when(codeSearcherMock.getRepoDocuments("testRepoName", 1)).thenReturn(repoReturn);
    when(codeSearcherMock.getRepoDocuments("testRepoName", 2)).thenReturn(new ArrayList<>());
    gitRepoJob.cleanMissingPathFiles(codeSearcherMock, "testRepoName", new HashMap<String, String>());
    verify(codeSearcherMock, times(1)).getRepoDocuments("testRepoName", 0);
    verify(codeSearcherMock, times(1)).getRepoDocuments("testRepoName", 1);
    verify(codeSearcherMock, times(1)).getRepoDocuments("testRepoName", 2);
}

Example 2 with CodeSearcher

use of com.searchcode.app.service.CodeSearcher in project searchcode-server by boyter.

the class EndToEndITCase method testEndToEndSvnPath.

public void testEndToEndSvnPath() throws IOException {
    CodeSearcher cs = new CodeSearcher();
    File directoryWithFiles = TestHelpers.createDirectoryWithFiles("EndToEndSvnTest");
    IndexSvnRepoJob indexSvnRepoJob = new IndexSvnRepoJob();
    indexSvnRepoJob.indexDocsByPath(Paths.get(directoryWithFiles.toString()), "ENDTOENDTEST", "", directoryWithFiles.toString(), false);
    SearchResult searchResult = cs.search("endtoendtestfile", 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(3);
    CodeResult codeResult1 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile1.php")).findFirst().get();
    CodeResult codeResult2 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile2.py")).findFirst().get();
    CodeResult codeResult3 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile3.java")).findFirst().get();
    assertThat(codeResult1.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile1");
    assertThat(codeResult2.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile2");
    assertThat(codeResult3.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile3");
    // Delete a single file
    String codeId = searchResult.getCodeResultList().get(0).getCodeId();
    Singleton.getCodeIndexer().deleteByCodeId(codeId);
    searchResult = cs.search("endtoendtestfile".toLowerCase(), 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(2);
    // Delete file from disk then index to ensure it is removed from the index
    File toDelete = new File(directoryWithFiles.toString() + "/EndToEndTestFile2.py");
    toDelete.delete();
    indexSvnRepoJob.indexDocsByPath(Paths.get(directoryWithFiles.toString()), "ENDTOENDTEST", "", directoryWithFiles.toString(), true);
    searchResult = cs.search("endtoendtestfile", 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(2);
    codeResult1 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile1.php")).findFirst().get();
    codeResult2 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile3.java")).findFirst().get();
    assertThat(codeResult1.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile1");
    assertThat(codeResult2.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile3");
    Singleton.getCodeIndexer().deleteByReponame("ENDTOENDTEST");
    searchResult = cs.search("endtoendtestfile".toLowerCase(), 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(0);
}

Example 3 with CodeSearcher

use of com.searchcode.app.service.CodeSearcher in project searchcode-server by boyter.

the class EndToEndITCase method testEndToEndFilePath.

public void testEndToEndFilePath() throws IOException {
    CodeSearcher cs = new CodeSearcher();
    File directoryWithFiles = TestHelpers.createDirectoryWithFiles("EndToEndFileTest");
    IndexFileRepoJob indexFileRepoJob = new IndexFileRepoJob();
    // Index created files
    indexFileRepoJob.indexDocsByPath(Paths.get(directoryWithFiles.toString()), "ENDTOENDTEST", "", directoryWithFiles.toString(), false);
    SearchResult searchResult = cs.search("endtoendtestfile", 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(3);
    CodeResult codeResult1 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile1.php")).findFirst().get();
    CodeResult codeResult2 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile2.py")).findFirst().get();
    CodeResult codeResult3 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile3.java")).findFirst().get();
    assertThat(codeResult1.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile1");
    assertThat(codeResult2.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile2");
    assertThat(codeResult3.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile3");
    // Delete a single file
    String codeId = searchResult.getCodeResultList().get(0).getCodeId();
    Singleton.getCodeIndexer().deleteByCodeId(codeId);
    searchResult = cs.search("endtoendtestfile".toLowerCase(), 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(2);
    codeResult1 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile2.py")).findFirst().get();
    codeResult2 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile3.java")).findFirst().get();
    assertThat(codeResult1.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile2");
    assertThat(codeResult2.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile3");
    // Delete file from disk then index to ensure it is removed from the index
    File toDelete = new File(directoryWithFiles.toString() + "/EndToEndTestFile2.py");
    toDelete.delete();
    indexFileRepoJob.indexDocsByPath(Paths.get(directoryWithFiles.toString()), "ENDTOENDTEST", "", directoryWithFiles.toString(), true);
    searchResult = cs.search("endtoendtestfile", 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(2);
    codeResult1 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile1.php")).findFirst().get();
    codeResult2 = searchResult.getCodeResultList().stream().filter(x -> x.getFileName().equals("EndToEndTestFile3.java")).findFirst().get();
    assertThat(codeResult1.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile1");
    assertThat(codeResult2.getCode().get(0)).isEqualTo("EndToEndTestFile EndToEndTestFile3");
    // Delete everything
    Singleton.getCodeIndexer().deleteByReponame("ENDTOENDTEST");
    searchResult = cs.search("endtoendtestfile".toLowerCase(), 0);
    assertThat(searchResult.getCodeResultList().size()).isEqualTo(0);
}

Example 4 with CodeSearcher

use of com.searchcode.app.service.CodeSearcher in project searchcode-server by boyter.

the class IndexBaseRepoJob method indexDocsByPath.

/**
     * Indexes all the documents in the path provided. Will also remove anything from the index if not on disk
     * Generally this is a slow update used only for the initial clone of a repository
     * NB this can be used for updates but it will be much slower as it needs to to walk the contents of the disk
     */
public void indexDocsByPath(Path path, String repoName, String repoLocations, String repoRemoteLocation, boolean existingRepo) {
    String fileRepoLocations = FilenameUtils.separatorsToUnix(repoLocations);
    SearchcodeFileVisitor<Path> searchcodeFileVisitor = new SearchcodeFileVisitor<>(this, repoName, fileRepoLocations, repoRemoteLocation);
    try {
        if (this.FOLLOWLINKS) {
            Files.walkFileTree(path, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, searchcodeFileVisitor);
        } else {
            Files.walkFileTree(path, searchcodeFileVisitor);
        }
    } catch (IOException ex) {
        Singleton.getLogger().warning("ERROR - caught a " + ex.getClass() + " in " + this.getClass() + " indexDocsByPath walkFileTree\n with message: " + ex.getMessage());
    }
    if (this.LOGINDEXED) {
        logIndexed(repoName, searchcodeFileVisitor.reportList);
    }
    if (existingRepo) {
        CodeSearcher codeSearcher = new CodeSearcher();
        this.cleanMissingPathFiles(codeSearcher, repoName, searchcodeFileVisitor.fileLocationsMap);
    }
}

Example 5 with CodeSearcher

use of com.searchcode.app.service.CodeSearcher in project searchcode-server by boyter.

the class CodeRouteService method html.

public ModelAndView html(Request request, Response response) {
    Repo repo = Singleton.getRepo();
    Data data = Singleton.getData();
    SearchcodeLib scl = Singleton.getSearchcodeLib(data);
    CodeSearcher cs = new CodeSearcher();
    CodeMatcher cm = new CodeMatcher(data);
    Map<String, Object> map = new HashMap<>();
    map.put("repoCount", repo.getRepoCount());
    if (request.queryParams().contains("q")) {
        String query = request.queryParams("q").trim();
        String altquery = query.replaceAll("[^A-Za-z0-9 ]", " ").trim().replaceAll(" +", " ");
        int page = 0;
        if (request.queryParams().contains("p")) {
            try {
                page = Integer.parseInt(request.queryParams("p"));
                page = page > 19 ? 19 : page;
            } catch (NumberFormatException ex) {
                page = 0;
            }
        }
        String[] repos = new String[0];
        String[] langs = new String[0];
        String[] owners = new String[0];
        String reposFilter = Values.EMPTYSTRING;
        String langsFilter = Values.EMPTYSTRING;
        String ownersFilter = Values.EMPTYSTRING;
        String reposQueryString = Values.EMPTYSTRING;
        String langsQueryString = Values.EMPTYSTRING;
        String ownsQueryString = Values.EMPTYSTRING;
        if (request.queryParams().contains("repo")) {
            repos = request.queryParamsValues("repo");
            if (repos.length != 0) {
                List<String> reposList = Arrays.asList(repos).stream().map((s) -> "reponame:" + QueryParser.escape(s)).collect(Collectors.toList());
                reposFilter = " && (" + StringUtils.join(reposList, " || ") + ")";
                List<String> reposQueryList = Arrays.asList(repos).stream().map((s) -> "&repo=" + URLEncoder.encode(s)).collect(Collectors.toList());
                reposQueryString = StringUtils.join(reposQueryList, "");
            }
        }
        if (request.queryParams().contains("lan")) {
            langs = request.queryParamsValues("lan");
            if (langs.length != 0) {
                List<String> langsList = Arrays.asList(langs).stream().map((s) -> "languagename:" + QueryParser.escape(s)).collect(Collectors.toList());
                langsFilter = " && (" + StringUtils.join(langsList, " || ") + ")";
                List<String> langsQueryList = Arrays.asList(langs).stream().map((s) -> "&lan=" + URLEncoder.encode(s)).collect(Collectors.toList());
                langsQueryString = StringUtils.join(langsQueryList, "");
            }
        }
        if (request.queryParams().contains("own")) {
            owners = request.queryParamsValues("own");
            if (owners.length != 0) {
                List<String> ownersList = Arrays.asList(owners).stream().map((s) -> "codeowner:" + QueryParser.escape(s)).collect(Collectors.toList());
                ownersFilter = " && (" + StringUtils.join(ownersList, " || ") + ")";
                List<String> ownsQueryList = Arrays.asList(owners).stream().map((s) -> "&own=" + URLEncoder.encode(s)).collect(Collectors.toList());
                ownsQueryString = StringUtils.join(ownsQueryList, "");
            }
        }
        // split the query escape it and and it together
        String cleanQueryString = scl.formatQueryString(query);
        SearchResult searchResult = cs.search(cleanQueryString + reposFilter + langsFilter + ownersFilter, page);
        searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), query, true));
        for (CodeFacetRepo f : searchResult.getRepoFacetResults()) {
            if (Arrays.asList(repos).contains(f.getRepoName())) {
                f.setSelected(true);
            }
        }
        for (CodeFacetLanguage f : searchResult.getLanguageFacetResults()) {
            if (Arrays.asList(langs).contains(f.getLanguageName())) {
                f.setSelected(true);
            }
        }
        for (CodeFacetOwner f : searchResult.getOwnerFacetResults()) {
            if (Arrays.asList(owners).contains(f.getOwner())) {
                f.setSelected(true);
            }
        }
        map.put("searchValue", query);
        map.put("searchResult", searchResult);
        map.put("reposQueryString", reposQueryString);
        map.put("langsQueryString", langsQueryString);
        map.put("ownsQueryString", ownsQueryString);
        map.put("altQuery", altquery);
        map.put("totalPages", searchResult.getPages().size());
        map.put("isHtml", true);
        map.put("logoImage", CommonRouteService.getLogo());
        map.put("isCommunity", App.ISCOMMUNITY);
        map.put(Values.EMBED, Singleton.getData().getDataByName(Values.EMBED, Values.EMPTYSTRING));
        return new ModelAndView(map, "searchresults.ftl");
    }
    map.put("photoId", CommonRouteService.getPhotoId());
    map.put("numDocs", cs.getTotalNumberDocumentsIndexed());
    map.put("logoImage", CommonRouteService.getLogo());
    map.put("isCommunity", App.ISCOMMUNITY);
    map.put(Values.EMBED, Singleton.getData().getDataByName(Values.EMBED, Values.EMPTYSTRING));
    return new ModelAndView(map, "index.ftl");
}