Examples with Project org.opengrok.indexer.configuration.Project used on opensource projects

Example 41 with Project

use of org.opengrok.indexer.configuration.Project in project OpenGrok by OpenGrok.

the class Indexer method main.

/**
 * Program entry point.
 *
 * @param argv argument vector
 */
@SuppressWarnings("PMD.UseStringBufferForStringAppends")
public static void main(String[] argv) {
    // this won't count JVM creation though
    Statistics stats = new Statistics();
    boolean update = true;
    Executor.registerErrorHandler();
    List<String> subFiles = RuntimeEnvironment.getInstance().getSubFiles();
    ArrayList<String> subFilesList = new ArrayList<>();
    boolean createDict = false;
    try {
        argv = parseOptions(argv);
        if (webappURI != null && !HostUtil.isReachable(webappURI, WEBAPP_CONNECT_TIMEOUT)) {
            System.err.println(webappURI + " is not reachable.");
            System.exit(1);
        }
        /*
             * Attend to disabledRepositories here in case exitWithHelp() will
             * need to report about repos.
             */
        disabledRepositories.addAll(cfg.getDisabledRepositories());
        cfg.setDisabledRepositories(disabledRepositories);
        for (String repoName : disabledRepositories) {
            LOGGER.log(Level.FINEST, "Disabled {0}", repoName);
        }
        if (help) {
            exitWithHelp();
        }
        checkConfiguration();
        if (awaitProfiler) {
            pauseToAwaitProfiler();
        }
        env = RuntimeEnvironment.getInstance();
        env.setIndexer(true);
        // Complete the configuration of repository types.
        List<Class<? extends Repository>> repositoryClasses = RepositoryFactory.getRepositoryClasses();
        for (Class<? extends Repository> clazz : repositoryClasses) {
            // Set external repository binaries from System properties.
            try {
                Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
                Object key = f.get(null);
                if (key != null) {
                    cfg.setRepoCmd(clazz.getCanonicalName(), System.getProperty(key.toString()));
                }
            } catch (Exception e) {
            // don't care
            }
        }
        // Logging starts here.
        if (verbose) {
            String fn = LoggerUtil.getFileHandlerPattern();
            if (fn != null) {
                System.out.println("Logging filehandler pattern: " + fn);
            }
        }
        // automatically allow symlinks that are directly in source root
        File sourceRootFile = new File(cfg.getSourceRoot());
        File[] projectDirs = sourceRootFile.listFiles();
        if (projectDirs != null) {
            for (File projectDir : projectDirs) {
                if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
                    allowedSymlinks.add(projectDir.getAbsolutePath());
                }
            }
        }
        allowedSymlinks.addAll(cfg.getAllowedSymlinks());
        cfg.setAllowedSymlinks(allowedSymlinks);
        canonicalRoots.addAll(cfg.getCanonicalRoots());
        cfg.setCanonicalRoots(canonicalRoots);
        // This will be used to perform more fine-grained checking in invalidateRepositories().
        for (String arg : argv) {
            String path = Paths.get(cfg.getSourceRoot(), arg).toString();
            subFilesList.add(path);
        }
        // according to the project key which is the same.
        for (Entry<String, Project> entry : cfg.getProjects().entrySet()) {
            if (entry.getValue().getName() == null) {
                entry.getValue().setName(entry.getKey());
            }
        }
        // with return code upon failure.
        if (checkIndex) {
            if (cfg.getDataRoot() == null || cfg.getDataRoot().isEmpty()) {
                System.err.println("Need data root in configuration for index check (use -R)");
                System.exit(1);
            }
            if (!IndexCheck.check(cfg, subFilesList)) {
                System.err.printf("Index check failed%n");
                System.err.print("You might want to remove " + (!subFilesList.isEmpty() ? "data for projects " + String.join(",", subFilesList) : "all data") + " under the data root and reindex\n");
                System.exit(1);
            }
            System.exit(0);
        }
        // Set updated configuration in RuntimeEnvironment.
        env.setConfiguration(cfg, subFilesList, CommandTimeoutType.INDEXER);
        // Let repository types to add items to ignoredNames.
        // This changes env so is called after the setConfiguration()
        // call above.
        RepositoryFactory.initializeIgnoredNames(env);
        if (bareConfig) {
            getInstance().sendToConfigHost(env, webappURI);
            writeConfigToFile(env, configFilename);
            System.exit(0);
        }
        /*
             * Add paths to directories under source root. If projects
             * are enabled the path should correspond to a project because
             * project path is necessary to correctly set index directory
             * (otherwise the index files will end up in index data root
             * directory and not per project data root directory).
             * For the check we need to have 'env' already set.
             */
        for (String path : subFilesList) {
            String srcPath = env.getSourceRootPath();
            if (srcPath == null) {
                System.err.println("Error getting source root from environment. Exiting.");
                System.exit(1);
            }
            path = path.substring(srcPath.length());
            if (env.hasProjects()) {
                // The paths need to correspond to a project.
                Project project;
                if ((project = Project.getProject(path)) != null) {
                    subFiles.add(path);
                    List<RepositoryInfo> repoList = env.getProjectRepositoriesMap().get(project);
                    if (repoList != null) {
                        repositories.addAll(repoList.stream().map(RepositoryInfo::getDirectoryNameRelative).collect(Collectors.toSet()));
                    }
                } else {
                    System.err.println("The path " + path + " does not correspond to a project");
                }
            } else {
                subFiles.add(path);
            }
        }
        if (!subFilesList.isEmpty() && subFiles.isEmpty()) {
            System.err.println("None of the paths were added, exiting");
            System.exit(1);
        }
        Metrics.updateSubFiles(subFiles);
        // emitted during indexing do not cause validation error.
        if (addProjects && webappURI != null) {
            try {
                IndexerUtil.enableProjects(webappURI);
            } catch (Exception e) {
                LOGGER.log(Level.SEVERE, String.format("Couldn't notify the webapp on %s.", webappURI), e);
                System.err.printf("Couldn't notify the webapp on %s: %s.%n", webappURI, e.getLocalizedMessage());
            }
        }
        LOGGER.log(Level.INFO, "Indexer version {0} ({1}) running on Java {2}", new Object[] { Info.getVersion(), Info.getRevision(), System.getProperty("java.version") });
        // Create history cache first.
        if (searchRepositories) {
            if (searchPaths.isEmpty()) {
                String[] dirs = env.getSourceRootFile().list((f, name) -> f.isDirectory() && env.getPathAccepter().accept(f));
                if (dirs != null) {
                    searchPaths.addAll(Arrays.asList(dirs));
                }
            }
            searchPaths = searchPaths.stream().map(t -> Paths.get(env.getSourceRootPath(), t).toString()).collect(Collectors.toSet());
        }
        getInstance().prepareIndexer(env, searchPaths, addProjects, createDict, runIndex, subFiles, new ArrayList<>(repositories));
        // prepareIndexer() populated the list of projects so now default projects can be set.
        env.setDefaultProjectsFromNames(defaultProjects);
        // And now index it all.
        if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
            IndexChangedListener progress = new DefaultIndexChangedListener();
            getInstance().doIndexerExecution(update, subFiles, progress);
        }
        writeConfigToFile(env, configFilename);
        // or send new configuration to the web application in the case of full reindex.
        if (webappURI != null) {
            if (!subFiles.isEmpty()) {
                getInstance().refreshSearcherManagers(env, subFiles, webappURI);
            } else {
                getInstance().sendToConfigHost(env, webappURI);
            }
        }
        env.getIndexerParallelizer().bounce();
    } catch (ParseException e) {
        System.err.println("** " + e.getMessage());
        System.exit(1);
    } catch (IndexerException ex) {
        LOGGER.log(Level.SEVERE, "Exception running indexer", ex);
        System.err.println("Exception: " + ex.getLocalizedMessage());
        System.err.println(optParser.getUsage());
        System.exit(1);
    } catch (Throwable e) {
        LOGGER.log(Level.SEVERE, "Unexpected Exception", e);
        System.err.println("Exception: " + e.getLocalizedMessage());
        System.exit(1);
    } finally {
        stats.report(LOGGER, "Indexer finished", "indexer.total");
    }
}

Example 42 with Project

use of org.opengrok.indexer.configuration.Project in project OpenGrok by OpenGrok.

the class Indexer method doIndexerExecution.

/**
 * This is the second phase of the indexer which generates Lucene index
 * by passing source code files through ctags, generating xrefs
 * and storing data from the source files in the index (along with history,
 * if any).
 *
 * @param update if set to true, index database is updated, otherwise optimized
 * @param subFiles index just some subdirectories
 * @param progress object to receive notifications as indexer progress is made
 * @throws IOException if I/O exception occurred
 */
public void doIndexerExecution(final boolean update, List<String> subFiles, IndexChangedListener progress) throws IOException {
    Statistics elapsed = new Statistics();
    LOGGER.info("Starting indexing");
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    IndexerParallelizer parallelizer = env.getIndexerParallelizer();
    final CountDownLatch latch;
    if (subFiles == null || subFiles.isEmpty()) {
        if (update) {
            latch = IndexDatabase.updateAll(progress);
        } else if (env.isOptimizeDatabase()) {
            latch = IndexDatabase.optimizeAll();
        } else {
            latch = new CountDownLatch(0);
        }
    } else {
        List<IndexDatabase> dbs = new ArrayList<>();
        for (String path : subFiles) {
            Project project = Project.getProject(path);
            if (project == null && env.hasProjects()) {
                LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
            } else {
                IndexDatabase db;
                if (project == null) {
                    db = new IndexDatabase();
                } else {
                    db = new IndexDatabase(project);
                }
                int idx = dbs.indexOf(db);
                if (idx != -1) {
                    db = dbs.get(idx);
                }
                if (db.addDirectory(path)) {
                    if (idx == -1) {
                        dbs.add(db);
                    }
                } else {
                    LOGGER.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
                }
            }
        }
        latch = new CountDownLatch(dbs.size());
        for (final IndexDatabase db : dbs) {
            final boolean optimize = env.isOptimizeDatabase();
            db.addIndexChangedListener(progress);
            parallelizer.getFixedExecutor().submit(() -> {
                try {
                    if (update) {
                        db.update();
                    } else if (optimize) {
                        db.optimize();
                    }
                } catch (Throwable e) {
                    LOGGER.log(Level.SEVERE, "An error occurred while " + (update ? "updating" : "optimizing") + " index", e);
                } finally {
                    latch.countDown();
                }
            });
        }
    }
    // Wait forever for the executors to finish.
    try {
        LOGGER.info("Waiting for the executors to finish");
        latch.await();
    } catch (InterruptedException exp) {
        LOGGER.log(Level.WARNING, "Received interrupt while waiting" + " for executor to finish", exp);
    }
    elapsed.report(LOGGER, "Done indexing data of all repositories", "indexer.repository.indexing");
    CtagsUtil.deleteTempFiles();
}

Example 43 with Project

use of org.opengrok.indexer.configuration.Project in project OpenGrok by OpenGrok.

the class IndexDatabase method listFrequentTokens.

static void listFrequentTokens(List<String> subFiles) throws IOException {
    final int limit = 4;
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.hasProjects()) {
        if (subFiles == null || subFiles.isEmpty()) {
            for (Project project : env.getProjectList()) {
                IndexDatabase db = new IndexDatabase(project);
                db.listTokens(limit);
            }
        } else {
            for (String path : subFiles) {
                Project project = Project.getProject(path);
                if (project == null) {
                    LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
                } else {
                    IndexDatabase db = new IndexDatabase(project);
                    db.listTokens(limit);
                }
            }
        }
    } else {
        IndexDatabase db = new IndexDatabase();
        db.listTokens(limit);
    }
}

Example 44 with Project

use of org.opengrok.indexer.configuration.Project in project OpenGrok by OpenGrok.

the class IndexDatabase method getAllFiles.

/**
 * Get all files in some of the index databases.
 *
 * @param subFiles Subdirectories of various projects or null or an empty list to get everything
 * @throws IOException if an error occurs
 * @return set of files in the index databases specified by the subFiles parameter
 */
public static Set<String> getAllFiles(List<String> subFiles) throws IOException {
    Set<String> files = new HashSet<>();
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.hasProjects()) {
        if (subFiles == null || subFiles.isEmpty()) {
            for (Project project : env.getProjectList()) {
                IndexDatabase db = new IndexDatabase(project);
                files.addAll(db.getFiles());
            }
        } else {
            for (String path : subFiles) {
                Project project = Project.getProject(path);
                if (project == null) {
                    LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
                } else {
                    IndexDatabase db = new IndexDatabase(project);
                    files.addAll(db.getFiles());
                }
            }
        }
    } else {
        IndexDatabase db = new IndexDatabase();
        files = db.getFiles();
    }
    return files;
}

Example 45 with Project

use of org.opengrok.indexer.configuration.Project in project OpenGrok by OpenGrok.

the class IndexDatabase method update.

/**
 * Update the index database for a number of sub-directories.
 *
 * @param listener where to signal the changes to the database
 * @param paths list of paths to be indexed
 */
public static void update(IndexChangedListener listener, List<String> paths) {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    IndexerParallelizer parallelizer = env.getIndexerParallelizer();
    List<IndexDatabase> dbs = new ArrayList<>();
    for (String path : paths) {
        Project project = Project.getProject(path);
        if (project == null && env.hasProjects()) {
            LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
        } else {
            IndexDatabase db;
            try {
                if (project == null) {
                    db = new IndexDatabase();
                } else {
                    db = new IndexDatabase(project);
                }
                int idx = dbs.indexOf(db);
                if (idx != -1) {
                    db = dbs.get(idx);
                }
                if (db.addDirectory(path)) {
                    if (idx == -1) {
                        dbs.add(db);
                    }
                } else {
                    LOGGER.log(Level.WARNING, "Directory does not exist \"{0}\" .", path);
                }
            } catch (IOException e) {
                LOGGER.log(Level.WARNING, "An error occurred while updating index", e);
            }
        }
        for (final IndexDatabase db : dbs) {
            db.addIndexChangedListener(listener);
            parallelizer.getFixedExecutor().submit(() -> {
                try {
                    db.update();
                } catch (Throwable e) {
                    LOGGER.log(Level.SEVERE, "An error occurred while updating index", e);
                }
            });
        }
    }
}