Examples with HistoryGuru org.opengrok.indexer.history.HistoryGuru used on opensource projects

Example 1 with HistoryGuru

use of org.opengrok.indexer.history.HistoryGuru in project OpenGrok by OpenGrok.

the class AnalyzerGuru method populateDocument.

/**
 * Populate a Lucene document with the required fields.
 *
 * @param doc The document to populate
 * @param file The file to index
 * @param path Where the file is located (from source root)
 * @param fa The analyzer to use on the file
 * @param xrefOut Where to write the xref (possibly {@code null})
 * @throws IOException If an exception occurs while collecting the data
 * @throws InterruptedException if a timeout occurs
 */
public void populateDocument(Document doc, File file, String path, AbstractAnalyzer fa, Writer xrefOut) throws IOException, InterruptedException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    path = Util.fixPathIfWindows(path);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
    if (RuntimeEnvironment.getInstance().isHistoryEnabled()) {
        try {
            HistoryGuru histGuru = HistoryGuru.getInstance();
            HistoryReader hr = histGuru.getHistoryReader(file);
            if (hr != null) {
                doc.add(new TextField(QueryBuilder.HIST, hr));
                History history;
                if ((history = histGuru.getHistory(file)) != null) {
                    List<HistoryEntry> historyEntries = history.getHistoryEntries(1, 0);
                    if (!historyEntries.isEmpty()) {
                        HistoryEntry histEntry = historyEntries.get(0);
                        doc.add(new TextField(QueryBuilder.LASTREV, histEntry.getRevision(), Store.YES));
                    }
                }
            }
        } catch (HistoryException e) {
            LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
        }
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    // `path' is not null, as it was passed to Util.path2uid() above.
    doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
    Project project = Project.getProject(path);
    if (project != null) {
        doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
    }
    /*
         * Use the parent of the path -- not the absolute file as is done for
         * FULLPATH -- so that DIRPATH is the same convention as for PATH
         * above. A StringField, however, is used instead of a TextField.
         */
    File fpath = new File(path);
    String fileParent = fpath.getParent();
    if (fileParent != null && fileParent.length() > 0) {
        String normalizedPath = QueryBuilder.normalizeDirPath(fileParent);
        StringField npstring = new StringField(QueryBuilder.DIRPATH, normalizedPath, Store.NO);
        doc.add(npstring);
    }
    if (fa != null) {
        AbstractAnalyzer.Genre g = fa.getGenre();
        if (g == AbstractAnalyzer.Genre.PLAIN || g == AbstractAnalyzer.Genre.XREFABLE || g == AbstractAnalyzer.Genre.HTML) {
            doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
        }
        fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
        String type = fa.getFileTypeName();
        doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
}

Example 2 with HistoryGuru

use of org.opengrok.indexer.history.HistoryGuru in project OpenGrok by OpenGrok.

the class ProjectsController method deleteHistoryCacheWorkHorse.

private void deleteHistoryCacheWorkHorse(String projectName, boolean clearHistoryGuru) {
    Project project = disableProject(projectName);
    LOGGER.log(Level.INFO, "deleting history cache for project {0}", projectName);
    List<RepositoryInfo> repos = env.getProjectRepositoriesMap().get(project);
    if (repos == null || repos.isEmpty()) {
        LOGGER.log(Level.INFO, "history cache for project {0} is not present", projectName);
        return;
    }
    // Delete history cache data.
    HistoryGuru guru = HistoryGuru.getInstance();
    guru.removeCache(repos.stream().map(x -> {
        try {
            return env.getPathRelativeToSourceRoot(new File((x).getDirectoryName()));
        } catch (ForbiddenSymlinkException e) {
            LOGGER.log(Level.FINER, e.getMessage());
            return "";
        } catch (IOException e) {
            LOGGER.log(Level.WARNING, "cannot remove files for repository {0}", x.getDirectoryName());
            // {@code removeCache()} will return nothing.
            return "";
        }
    }).filter(x -> !x.isEmpty()).collect(Collectors.toSet()), clearHistoryGuru);
}

Example 3 with HistoryGuru

use of org.opengrok.indexer.history.HistoryGuru in project OpenGrok by OpenGrok.

the class ProjectsControllerTest method testAdd.

/**
 * Verify that added project correctly inherits a property
 * from configuration. Ideally, this should test all properties of Project.
 */
@Test
void testAdd() throws Exception {
    assertTrue(env.getRepositories().isEmpty());
    assertTrue(env.getProjects().isEmpty());
    // Add a group matching the project to be added.
    String groupName = "mercurialgroup";
    Group group = new Group(groupName, "mercurial.*");
    env.getGroups().add(group);
    assertTrue(env.hasGroups());
    assertEquals(1, env.getGroups().stream().filter(g -> g.getName().equals(groupName)).collect(Collectors.toSet()).size());
    assertEquals(0, group.getRepositories().size());
    assertEquals(0, group.getProjects().size());
    // Add a sub-repository.
    String repoPath = repository.getSourceRoot() + File.separator + "mercurial";
    File mercurialRoot = new File(repoPath);
    File subDir = new File(mercurialRoot, "usr");
    assertTrue(subDir.mkdir());
    String subRepoPath = repoPath + File.separator + "usr" + File.separator + "closed";
    File mercurialSubRoot = new File(subRepoPath);
    MercurialRepositoryTest.runHgCommand(mercurialRoot, "clone", mercurialRoot.getAbsolutePath(), subRepoPath);
    // Add the project.
    env.setScanningDepth(3);
    addProject("mercurial");
    // Check that the project was added properly.
    assertTrue(env.getProjects().containsKey("mercurial"));
    assertEquals(1, env.getProjects().size());
    assertEquals(2, env.getRepositories().size());
    assertEquals(1, group.getRepositories().size());
    assertEquals(0, group.getProjects().size());
    assertEquals(1, group.getRepositories().stream().filter(p -> p.getName().equals("mercurial")).collect(Collectors.toSet()).size());
    // Check that HistoryGuru now includes the project in its list.
    Set<String> directoryNames = HistoryGuru.getInstance().getRepositories().stream().map(RepositoryInfo::getDirectoryName).collect(Collectors.toSet());
    assertTrue(directoryNames.contains(repoPath) || directoryNames.contains(mercurialRoot.getCanonicalPath()), "though it should contain the top root,");
    assertTrue(directoryNames.contains(subRepoPath) || directoryNames.contains(mercurialSubRoot.getCanonicalPath()), "though it should contain the sub-root,");
    // Add more projects and check that they have been added incrementally.
    // At the same time, it checks that multiple projects can be added
    // with single message.
    addProject("git");
    assertEquals(2, env.getProjects().size());
    assertEquals(3, env.getRepositories().size());
    assertTrue(env.getProjects().containsKey("git"));
    assertFalse(HistoryGuru.getInstance().getRepositories().stream().map(RepositoryInfo::getDirectoryName).collect(Collectors.toSet()).contains("git"));
}

Example 4 with HistoryGuru

use of org.opengrok.indexer.history.HistoryGuru in project OpenGrok by OpenGrok.

the class ProjectsControllerTest method testDelete.

/**
 * This test needs to perform indexing so that it can be verified that
 * delete handling does remove the index data.
 */
@Test
void testDelete() throws Exception {
    String[] projectsToDelete = { "git" };
    // Add a group matching the project to be added.
    String groupName = "gitgroup";
    Group group = new Group(groupName, "git.*");
    env.getGroups().add(group);
    assertTrue(env.hasGroups());
    assertEquals(1, env.getGroups().stream().filter(g -> g.getName().equals(groupName)).collect(Collectors.toSet()).size());
    assertEquals(0, group.getRepositories().size());
    assertEquals(0, group.getProjects().size());
    assertEquals(0, env.getProjects().size());
    assertEquals(0, env.getRepositories().size());
    assertEquals(0, env.getProjectRepositoriesMap().size());
    addProject("mercurial");
    addProject("git");
    assertEquals(2, env.getProjects().size());
    assertEquals(2, env.getRepositories().size());
    assertEquals(2, env.getProjectRepositoriesMap().size());
    // Check the group was populated properly.
    assertEquals(1, group.getRepositories().size());
    assertEquals(0, group.getProjects().size());
    assertEquals(1, group.getRepositories().stream().filter(p -> p.getName().equals("git")).collect(Collectors.toSet()).size());
    // Run the indexer so that data directory is populated.
    ArrayList<String> subFiles = new ArrayList<>();
    subFiles.add("/git");
    subFiles.add("/mercurial");
    ArrayList<String> repos = new ArrayList<>();
    repos.add("/git");
    repos.add("/mercurial");
    // This is necessary so that repositories in HistoryGuru get populated.
    // For per project reindex this is called from setConfiguration() because
    // of the -R option is present.
    HistoryGuru.getInstance().invalidateRepositories(env.getRepositories(), null, CommandTimeoutType.INDEXER);
    env.setHistoryEnabled(true);
    Indexer.getInstance().prepareIndexer(env, // don't search for repositories
    false, // don't scan and add projects
    false, // don't create dictionary
    false, // subFiles - needed when refreshing history partially
    subFiles, // repositories - needed when refreshing history partially
    repos);
    Indexer.getInstance().doIndexerExecution(true, null, null);
    for (String proj : projectsToDelete) {
        deleteProject(proj);
    }
    assertEquals(1, env.getProjects().size());
    assertEquals(1, env.getRepositories().size());
    assertEquals(1, env.getProjectRepositoriesMap().size());
    // Test data removal.
    for (String projectName : projectsToDelete) {
        for (String dirName : new String[] { "historycache", IndexDatabase.XREF_DIR, IndexDatabase.INDEX_DIR }) {
            File dir = new File(env.getDataRootFile(), dirName + File.separator + projectName);
            assertFalse(dir.exists());
        }
    }
    // Check that HistoryGuru no longer maintains the removed projects.
    for (String p : projectsToDelete) {
        assertFalse(HistoryGuru.getInstance().getRepositories().stream().map(RepositoryInfo::getDirectoryName).collect(Collectors.toSet()).contains(repository.getSourceRoot() + File.separator + p));
    }
    // Check the group no longer contains the removed project.
    assertEquals(0, group.getRepositories().size());
    assertEquals(0, group.getProjects().size());
}

Example 5 with HistoryGuru

use of org.opengrok.indexer.history.HistoryGuru in project OpenGrok by OpenGrok.

the class RuntimeEnvironment method setConfiguration.

/**
 * Sets the configuration and performs necessary actions.
 *
 * @param configuration new configuration
 * @param subFileList   list of repositories
 * @param cmdType       command timeout type
 */
public synchronized void setConfiguration(Configuration configuration, List<String> subFileList, CommandTimeoutType cmdType) {
    try (ResourceLock resourceLock = configLock.writeLockAsResource()) {
        // noinspection ConstantConditions to avoid warning of no reference to auto-closeable
        assert resourceLock != null;
        this.configuration = configuration;
    }
    // HistoryGuru constructor needs environment properties so no locking is done here.
    HistoryGuru histGuru = HistoryGuru.getInstance();
    // Set the working repositories in HistoryGuru.
    if (subFileList != null) {
        histGuru.invalidateRepositories(getRepositories(), subFileList, cmdType);
    } else {
        histGuru.invalidateRepositories(getRepositories(), cmdType);
    }
    // The invalidation of repositories above might have excluded some
    // repositories in HistoryGuru so the configuration needs to reflect that.
    setRepositories(new ArrayList<>(histGuru.getRepositories()));
    // generate repository map is dependent on getRepositories()
    try {
        generateProjectRepositoriesMap();
    } catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Cannot generate project - repository map", ex);
    }
    // populate groups is dependent on repositories map
    populateGroups(getGroups(), new TreeSet<>(getProjects().values()));
    includeFiles.reloadIncludeFiles();
}