Examples with RuntimeEnvironment org.opengrok.indexer.configuration.RuntimeEnvironment used on opensource projects

Example 11 with RuntimeEnvironment

use of org.opengrok.indexer.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.

the class IndexDatabase method getAllFiles.

/**
 * Get all files in some of the index databases.
 *
 * @param subFiles Subdirectories of various projects or null or an empty list to get everything
 * @throws IOException if an error occurs
 * @return set of files in the index databases specified by the subFiles parameter
 */
public static Set<String> getAllFiles(List<String> subFiles) throws IOException {
    Set<String> files = new HashSet<>();
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.hasProjects()) {
        if (subFiles == null || subFiles.isEmpty()) {
            for (Project project : env.getProjectList()) {
                IndexDatabase db = new IndexDatabase(project);
                files.addAll(db.getFiles());
            }
        } else {
            for (String path : subFiles) {
                Project project = Project.getProject(path);
                if (project == null) {
                    LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
                } else {
                    IndexDatabase db = new IndexDatabase(project);
                    files.addAll(db.getFiles());
                }
            }
        }
    } else {
        IndexDatabase db = new IndexDatabase();
        files = db.getFiles();
    }
    return files;
}

Example 12 with RuntimeEnvironment

use of org.opengrok.indexer.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.

the class IndexDatabase method addFile.

/**
 * Add a file to the Lucene index (and generate a xref file).
 *
 * @param file The file to add
 * @param path The path to the file (from source root)
 * @param ctags a defined instance to use (only if its binary is not null)
 * @throws java.io.IOException if an error occurs
 * @throws InterruptedException if a timeout occurs
 */
private void addFile(File file, String path, Ctags ctags) throws IOException, InterruptedException {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    AbstractAnalyzer fa = getAnalyzerFor(file, path);
    for (IndexChangedListener listener : listeners) {
        listener.fileAdd(path, fa.getClass().getSimpleName());
    }
    ctags.setTabSize(project != null ? project.getTabSize() : 0);
    if (env.getCtagsTimeout() != 0) {
        ctags.setTimeout(env.getCtagsTimeout());
    }
    fa.setCtags(ctags);
    fa.setCountsAggregator(countsAggregator);
    fa.setProject(Project.getProject(path));
    fa.setScopesEnabled(env.isScopesEnabled());
    fa.setFoldingEnabled(env.isFoldingEnabled());
    Document doc = new Document();
    CountingWriter xrefOut = null;
    try {
        String xrefAbs = null;
        File transientXref = null;
        if (env.isGenerateHtml()) {
            xrefAbs = getXrefPath(path);
            transientXref = new File(TandemPath.join(xrefAbs, PendingFileCompleter.PENDING_EXTENSION));
            xrefOut = newXrefWriter(path, transientXref, env.isCompressXref());
        }
        analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
        // Avoid producing empty xref files.
        if (xrefOut != null && xrefOut.getCount() > 0) {
            PendingFileRenaming ren = new PendingFileRenaming(xrefAbs, transientXref.getAbsolutePath());
            completer.add(ren);
        } else if (xrefOut != null) {
            LOGGER.log(Level.FINER, "xref for {0} would be empty, will remove", path);
            completer.add(new PendingFileDeletion(transientXref.toString()));
        }
    } catch (InterruptedException e) {
        LOGGER.log(Level.WARNING, "File ''{0}'' interrupted--{1}", new Object[] { path, e.getMessage() });
        cleanupResources(doc);
        throw e;
    } catch (Exception e) {
        LOGGER.log(Level.INFO, "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.", path);
        if (LOGGER.isLoggable(Level.FINE)) {
            LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e);
        }
        cleanupResources(doc);
        return;
    } finally {
        fa.setCtags(null);
        fa.setCountsAggregator(null);
        if (xrefOut != null) {
            xrefOut.close();
        }
    }
    try {
        writer.addDocument(doc);
    } catch (Throwable t) {
        cleanupResources(doc);
        throw t;
    }
    setDirty();
    for (IndexChangedListener listener : listeners) {
        listener.fileAdded(path, fa.getClass().getSimpleName());
    }
}

Example 13 with RuntimeEnvironment

use of org.opengrok.indexer.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.

the class IndexDatabase method acceptSymlink.

/**
 * Check if I should accept the path containing a symlink.
 *
 * @param absolute the path with a symlink to check
 * @param canonical the canonical file object
 * @param ret defined instance whose {@code localRelPath} property will be
 * non-null afterward if and only if {@code absolute} is a symlink that
 * targets either a {@link Repository}-local filesystem object or the same
 * object ({@code canonical}) as a previously-detected and allowed symlink.
 * N.b. method will return {@code false} if {@code ret.localRelPath} is set
 * non-null.
 * @return a value indicating if {@code file} should be included in index
 */
private boolean acceptSymlink(Path absolute, File canonical, AcceptSymlinkRet ret) {
    ret.localRelPath = null;
    String absolute1 = absolute.toString();
    String canonical1 = canonical.getPath();
    boolean isCanonicalDir = canonical.isDirectory();
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    IndexedSymlink indexed1;
    String absolute0;
    if (isLocal(canonical1)) {
        if (!isCanonicalDir) {
            if (LOGGER.isLoggable(Level.FINEST)) {
                LOGGER.log(Level.FINEST, "Local {0} has symlink from {1}", new Object[] { canonical1, absolute1 });
            }
            /*
                 * Always index symlinks to local files, but do not add to
                 * indexedSymlinks for a non-directory.
                 */
            return true;
        }
        /*
             * Do not index symlinks to local directories, because the
             * canonical target will be indexed on its own -- but relativize()
             * a path to be returned in ret so that a symlink can be replicated
             * in xref/.
             */
        ret.localRelPath = absolute.getParent().relativize(canonical.toPath()).toString();
        // Try to put the prime absolute path into indexedSymlinks.
        try {
            String primeRelative = env.getPathRelativeToSourceRoot(canonical);
            absolute0 = env.getSourceRootPath() + primeRelative;
        } catch (ForbiddenSymlinkException | IOException e) {
            /*
                 * This is not expected, as indexDown() would have operated on
                 * the file already -- but we are forced to handle.
                 */
            LOGGER.log(Level.WARNING, String.format("Unexpected error getting relative for %s", canonical), e);
            absolute0 = absolute1;
        }
        indexed1 = new IndexedSymlink(absolute0, canonical1, true);
        indexedSymlinks.put(canonical1, indexed1);
        return false;
    }
    IndexedSymlink indexed0;
    if ((indexed0 = indexedSymlinks.get(canonical1)) != null) {
        if (absolute1.equals(indexed0.getAbsolute())) {
            return true;
        }
        /*
             * Do not index symlinks to external directories already indexed
             * as linked elsewhere, because the canonical target will be
             * indexed already -- but relativize() a path to be returned in ret
             * so that this second symlink can be redone as a local
             * (non-external) symlink in xref/.
             */
        ret.localRelPath = absolute.getParent().relativize(Paths.get(indexed0.getAbsolute())).toString();
        if (LOGGER.isLoggable(Level.FINEST)) {
            LOGGER.log(Level.FINEST, "External dir {0} has symlink from {1} after first {2}", new Object[] { canonical1, absolute1, indexed0.getAbsolute() });
        }
        return false;
    }
    /*
         * Iterate through indexedSymlinks, which is sorted so that shorter
         * canonical entries come first, to see if the new link is a child
         * canonically.
         */
    for (IndexedSymlink a0 : indexedSymlinks.values()) {
        indexed0 = a0;
        if (!indexed0.isLocal() && canonical1.startsWith(indexed0.getCanonicalSeparated())) {
            absolute0 = indexed0.getAbsolute();
            if (!isCanonicalDir) {
                if (LOGGER.isLoggable(Level.FINEST)) {
                    LOGGER.log(Level.FINEST, "External file {0} has symlink from {1} under previous {2}", new Object[] { canonical1, absolute1, absolute0 });
                }
                // Do not add to indexedSymlinks for a non-directory.
                return true;
            }
            /*
                 * See above about redoing a sourceRoot symlink as a local
                 * (non-external) symlink in xref/.
                 */
            Path abs0 = Paths.get(absolute0, canonical1.substring(indexed0.getCanonicalSeparated().length()));
            ret.localRelPath = absolute.getParent().relativize(abs0).toString();
            if (LOGGER.isLoggable(Level.FINEST)) {
                LOGGER.log(Level.FINEST, "External dir {0} has symlink from {1} under previous {2}", new Object[] { canonical1, absolute1, absolute0 });
            }
            return false;
        }
    }
    Set<String> canonicalRoots = env.getCanonicalRoots();
    for (String canonicalRoot : canonicalRoots) {
        if (canonical1.startsWith(canonicalRoot)) {
            if (LOGGER.isLoggable(Level.FINEST)) {
                LOGGER.log(Level.FINEST, "Allowed symlink {0} per canonical root {1}", new Object[] { absolute1, canonical1 });
            }
            if (isCanonicalDir) {
                indexed1 = new IndexedSymlink(absolute1, canonical1, false);
                indexedSymlinks.put(canonical1, indexed1);
            }
            return true;
        }
    }
    Set<String> allowedSymlinks = env.getAllowedSymlinks();
    for (String allowedSymlink : allowedSymlinks) {
        String allowedTarget;
        try {
            allowedTarget = new File(allowedSymlink).getCanonicalPath();
        } catch (IOException e) {
            LOGGER.log(Level.FINE, "unresolvable symlink: {0}", allowedSymlink);
            continue;
        }
        /*
             * The following canonical check is sufficient because indexDown()
             * traverses top-down, and any intermediate symlinks would have
             * also been checked here for an allowed canonical match. This
             * technically means that if there is a set of redundant symlinks
             * with the same canonical target, then allowing one of the set
             * will allow all others in the set.
             */
        if (canonical1.equals(allowedTarget)) {
            if (isCanonicalDir) {
                indexed1 = new IndexedSymlink(absolute1, canonical1, false);
                indexedSymlinks.put(canonical1, indexed1);
            }
            return true;
        }
    }
    return false;
}

Example 14 with RuntimeEnvironment

use of org.opengrok.indexer.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.

the class IndexDatabase method isLocal.

/**
 * Check if a file is local to the current project. If we don't have
 * projects, check if the file is in the source root.
 *
 * @param path the path to a file
 * @return true if the file is local to the current repository
 */
private boolean isLocal(String path) {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    String srcRoot = env.getSourceRootPath();
    if (path.startsWith(srcRoot + File.separator)) {
        if (env.hasProjects()) {
            String relPath = path.substring(srcRoot.length());
            // If file is under the current project, then it's local.
            return project.equals(Project.getProject(relPath));
        } else {
            // consider it local.
            return true;
        }
    }
    return false;
}

Example 15 with RuntimeEnvironment

use of org.opengrok.indexer.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.

the class IndexDatabase method update.

/**
 * Update the index database for a number of sub-directories.
 *
 * @param listener where to signal the changes to the database
 * @param paths list of paths to be indexed
 */
public static void update(IndexChangedListener listener, List<String> paths) {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    IndexerParallelizer parallelizer = env.getIndexerParallelizer();
    List<IndexDatabase> dbs = new ArrayList<>();
    for (String path : paths) {
        Project project = Project.getProject(path);
        if (project == null && env.hasProjects()) {
            LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
        } else {
            IndexDatabase db;
            try {
                if (project == null) {
                    db = new IndexDatabase();
                } else {
                    db = new IndexDatabase(project);
                }
                int idx = dbs.indexOf(db);
                if (idx != -1) {
                    db = dbs.get(idx);
                }
                if (db.addDirectory(path)) {
                    if (idx == -1) {
                        dbs.add(db);
                    }
                } else {
                    LOGGER.log(Level.WARNING, "Directory does not exist \"{0}\" .", path);
                }
            } catch (IOException e) {
                LOGGER.log(Level.WARNING, "An error occurred while updating index", e);
            }
        }
        for (final IndexDatabase db : dbs) {
            db.addIndexChangedListener(listener);
            parallelizer.getFixedExecutor().submit(() -> {
                try {
                    db.update();
                } catch (Throwable e) {
                    LOGGER.log(Level.SEVERE, "An error occurred while updating index", e);
                }
            });
        }
    }
}