Examples with Repository org.opengrok.indexer.history.Repository used on opensource projects

Example 1 with Repository

use of org.opengrok.indexer.history.Repository in project OpenGrok by OpenGrok.

the class ProjectsController method deleteHistoryCacheWorkHorse.

private void deleteHistoryCacheWorkHorse(String projectName, boolean clearHistoryGuru) {
    Project project = disableProject(projectName);
    LOGGER.log(Level.INFO, "deleting history cache for project {0}", projectName);
    List<RepositoryInfo> repos = env.getProjectRepositoriesMap().get(project);
    if (repos == null || repos.isEmpty()) {
        LOGGER.log(Level.INFO, "history cache for project {0} is not present", projectName);
        return;
    }
    // Delete history cache data.
    HistoryGuru guru = HistoryGuru.getInstance();
    guru.removeCache(repos.stream().map(x -> {
        try {
            return env.getPathRelativeToSourceRoot(new File((x).getDirectoryName()));
        } catch (ForbiddenSymlinkException e) {
            LOGGER.log(Level.FINER, e.getMessage());
            return "";
        } catch (IOException e) {
            LOGGER.log(Level.WARNING, "cannot remove files for repository {0}", x.getDirectoryName());
            // {@code removeCache()} will return nothing.
            return "";
        }
    }).filter(x -> !x.isEmpty()).collect(Collectors.toSet()), clearHistoryGuru);
}

Example 2 with Repository

use of org.opengrok.indexer.history.Repository in project OpenGrok by OpenGrok.

the class Indexer method parseOptions.

/**
 * Parse OpenGrok Indexer options
 * This method was created so that it would be easier to write unit
 * tests against the Indexer option parsing mechanism.
 *
 * @param argv the command line arguments
 * @return array of remaining non option arguments
 * @throws ParseException if parsing failed
 */
public static String[] parseOptions(String[] argv) throws ParseException {
    final String[] usage = { HELP_OPT_1 };
    if (argv.length == 0) {
        // will force usage output
        argv = usage;
        // with non-zero EXIT STATUS
        status = 1;
    }
    /*
         * Pre-match any of the --help options so that some possible exception-generating args handlers (e.g. -R)
         * can be short-circuited.
         */
    boolean preHelp = Arrays.stream(argv).anyMatch(s -> HELP_OPT_1.equals(s) || HELP_OPT_2.equals(s) || HELP_OPT_3.equals(s));
    OptionParser configure = OptionParser.scan(parser -> parser.on("-R configPath").execute(cfgFile -> {
        try {
            cfg = Configuration.read(new File((String) cfgFile));
        } catch (IOException e) {
            if (!preHelp) {
                die(e.getMessage());
            } else {
                System.err.printf("Warning: failed to read -R %s%n", cfgFile);
            }
        }
    }));
    searchPaths.clear();
    // Limit usage lines to 72 characters for concise formatting.
    optParser = OptionParser.execute(parser -> {
        parser.setPrologue(String.format("%nUsage: java -jar %s [options] [subDir1 [...]]%n", OPENGROK_JAR));
        parser.on(HELP_OPT_3, HELP_OPT_2, HELP_OPT_1, "=[mode]", "With no mode specified, display this usage summary. Or specify a mode:", "  config - display configuration.xml examples.", "   ctags - display ctags command-line.", "    guru - display AnalyzerGuru details.", "   repos - display enabled repositories.").execute(v -> {
            help = true;
            helpUsage = parser.getUsage();
            String mode = (String) v;
            if (mode != null && !mode.isEmpty()) {
                try {
                    helpMode = HelpMode.valueOf(((String) v).toUpperCase(Locale.ROOT));
                } catch (IllegalArgumentException ex) {
                    die("mode '" + v + "' is not valid.");
                }
            }
        });
        parser.on("--apiTimeout", "=number", Integer.class, "Set timeout for asynchronous API requests.").execute(v -> cfg.setApiTimeout((Integer) v));
        parser.on("--connectTimeout", "=number", Integer.class, "Set connect timeout. Used for API requests.").execute(v -> cfg.setConnectTimeout((Integer) v));
        parser.on("-A (.ext|prefix.):(-|analyzer)", "--analyzer", "/(\\.\\w+|\\w+\\.):(-|[a-zA-Z_0-9.]+)/", "Associates files with the specified prefix or extension (case-", "insensitive) to be analyzed with the given analyzer, where 'analyzer'", "may be specified using a class name (case-sensitive e.g. RubyAnalyzer)", "or analyzer language name (case-sensitive e.g. C). Option may be", "repeated.", "  Ex: -A .foo:CAnalyzer", "      will use the C analyzer for all files ending with .FOO", "  Ex: -A bar.:Perl", "      will use the Perl analyzer for all files starting with", "      \"BAR\" (no full-stop)", "  Ex: -A .c:-", "      will disable specialized analyzers for all files ending with .c").execute(analyzerSpec -> {
            String[] arg = ((String) analyzerSpec).split(":");
            String fileSpec = arg[0];
            String analyzer = arg[1];
            configureFileAnalyzer(fileSpec, analyzer);
        });
        parser.on("-c", "--ctags", "=/path/to/ctags", "Path to Universal Ctags. Default is ctags in environment PATH.").execute(v -> cfg.setCtags((String) v));
        parser.on("--canonicalRoot", "=/path/", "Allow symlinks to canonical targets starting with the specified root", "without otherwise needing to specify -N,--symlink for such symlinks. A", "canonical root must end with a file separator. For security, a canonical", "root cannot be the root directory. Option may be repeated.").execute(v -> {
            String root = (String) v;
            String problem = CanonicalRootValidator.validate(root, "--canonicalRoot");
            if (problem != null) {
                die(problem);
            }
            canonicalRoots.add(root);
        });
        parser.on("--checkIndex", "Check index, exit with 0 on success,", "with 1 on failure.").execute(v -> checkIndex = true);
        parser.on("-d", "--dataRoot", "=/path/to/data/root", "The directory where OpenGrok stores the generated data.").execute(drPath -> {
            File dataRoot = new File((String) drPath);
            if (!dataRoot.exists() && !dataRoot.mkdirs()) {
                die("Cannot create data root: " + dataRoot);
            }
            if (!dataRoot.isDirectory()) {
                die("Data root must be a directory");
            }
            try {
                cfg.setDataRoot(dataRoot.getCanonicalPath());
            } catch (IOException e) {
                die(e.getMessage());
            }
        });
        parser.on("--depth", "=number", Integer.class, "Scanning depth for repositories in directory structure relative to", "source root. Default is " + Configuration.defaultScanningDepth + ".").execute(depth -> cfg.setScanningDepth((Integer) depth));
        parser.on("--disableRepository", "=type_name", "Disables operation of an OpenGrok-supported repository. See also", "-h,--help repos. Option may be repeated.", "  Ex: --disableRepository git", "      will disable the GitRepository", "  Ex: --disableRepository MercurialRepository").execute(v -> {
            String repoType = (String) v;
            String repoSimpleType = RepositoryFactory.matchRepositoryByName(repoType);
            if (repoSimpleType == null) {
                System.err.printf("'--disableRepository %s' does not match a type and is ignored%n", v);
            } else {
                disabledRepositories.add(repoSimpleType);
            }
        });
        parser.on("-e", "--economical", "To consume less disk space, OpenGrok will not generate and save", "hypertext cross-reference files but will generate on demand, which could", "be slightly slow.").execute(v -> cfg.setGenerateHtml(false));
        parser.on("-G", "--assignTags", "Assign commit tags to all entries in history for all repositories.").execute(v -> cfg.setTagsEnabled(true));
        parser.on("-H", "--history", "Enable history.").execute(v -> cfg.setHistoryEnabled(true));
        parser.on("--historyThreads", "=number", Integer.class, "The number of threads to use for history cache generation on repository level. " + "By default the number of threads will be set to the number of available CPUs.", "Assumes -H/--history.").execute(threadCount -> cfg.setHistoryParallelism((Integer) threadCount));
        parser.on("--historyFileThreads", "=number", Integer.class, "The number of threads to use for history cache generation when dealing with individual files.", "By default the number of threads will be set to the number of available CPUs.", "Assumes -H/--history.").execute(threadCount -> cfg.setHistoryFileParallelism((Integer) threadCount));
        parser.on("-I", "--include", "=pattern", "Only files matching this pattern will be examined. Pattern supports", "wildcards (example: -I '*.java' -I '*.c'). Option may be repeated.").execute(pattern -> cfg.getIncludedNames().add((String) pattern));
        parser.on("-i", "--ignore", "=pattern", "Ignore matching files (prefixed with 'f:' or no prefix) or directories", "(prefixed with 'd:'). Pattern supports wildcards (example: -i '*.so'", "-i d:'test*'). Option may be repeated.").execute(pattern -> cfg.getIgnoredNames().add((String) pattern));
        parser.on("-l", "--lock", "=on|off|simple|native", LUCENE_LOCKS, "Set OpenGrok/Lucene locking mode of the Lucene database during index", "generation. \"on\" is an alias for \"simple\". Default is off.").execute(v -> {
            try {
                if (v != null) {
                    String vuc = v.toString().toUpperCase(Locale.ROOT);
                    cfg.setLuceneLocking(LuceneLockName.valueOf(vuc));
                }
            } catch (IllegalArgumentException e) {
                System.err.printf("`--lock %s' is invalid and ignored%n", v);
            }
        });
        parser.on("--leadingWildCards", "=on|off", ON_OFF, Boolean.class, "Allow or disallow leading wildcards in a search. Default is on.").execute(v -> cfg.setAllowLeadingWildcard((Boolean) v));
        parser.on("-m", "--memory", "=number", Double.class, "Amount of memory (MB) that may be used for buffering added documents and", "deletions before they are flushed to the directory (default " + Configuration.defaultRamBufferSize + ").", "Please increase JVM heap accordingly too.").execute(memSize -> cfg.setRamBufferSize((Double) memSize));
        parser.on("--mandoc", "=/path/to/mandoc", "Path to mandoc(1) binary.").execute(mandocPath -> cfg.setMandoc((String) mandocPath));
        parser.on("-N", "--symlink", "=/path/to/symlink", "Allow the symlink to be followed. Other symlinks targeting the same", "canonical target or canonical children will be allowed too. Option may", "be repeated. (By default only symlinks directly under the source root", "directory are allowed. See also --canonicalRoot)").execute(v -> allowedSymlinks.add((String) v));
        parser.on("-n", "--noIndex", "Do not generate indexes and other data (such as history cache and xref", "files), but process all other command line options.").execute(v -> runIndex = false);
        parser.on("--nestingMaximum", "=number", Integer.class, "Maximum depth of nested repositories. Default is 1.").execute(v -> cfg.setNestingMaximum((Integer) v));
        parser.on("-O", "--optimize", "=on|off", ON_OFF, Boolean.class, "Turn on/off the optimization of the index database as part of the", "indexing step. Default is on.").execute(v -> {
            boolean oldval = cfg.isOptimizeDatabase();
            cfg.setOptimizeDatabase((Boolean) v);
            if (oldval != cfg.isOptimizeDatabase()) {
                optimizedChanged = true;
            }
        });
        parser.on("-o", "--ctagOpts", "=path", "File with extra command line options for ctags.").execute(path -> {
            String CTagsExtraOptionsFile = (String) path;
            File CTagsFile = new File(CTagsExtraOptionsFile);
            if (!(CTagsFile.isFile() && CTagsFile.canRead())) {
                die("File '" + CTagsExtraOptionsFile + "' not found for the -o option");
            }
            System.err.println("INFO: file with extra " + "options for ctags: " + CTagsExtraOptionsFile);
            cfg.setCTagsExtraOptionsFile(CTagsExtraOptionsFile);
        });
        parser.on("-P", "--projects", "Generate a project for each top-level directory in source root.").execute(v -> {
            addProjects = true;
            cfg.setProjectsEnabled(true);
        });
        parser.on("-p", "--defaultProject", "=path/to/default/project", "Path (relative to the source root) to a project that should be selected", "by default in the web application (when no other project is set either", "in a cookie or in parameter). Option may be repeated to specify several", "projects. Use the special value __all__ to indicate all projects.").execute(v -> defaultProjects.add((String) v));
        parser.on("--profiler", "Pause to await profiler or debugger.").execute(v -> awaitProfiler = true);
        parser.on("--progress", "Print per-project percentage progress information.").execute(v -> cfg.setPrintProgress(true));
        parser.on("-Q", "--quickScan", "=on|off", ON_OFF, Boolean.class, "Turn on/off quick context scan. By default, only the first 1024KB of a", "file is scanned, and a link ('[..all..]') is inserted when the file is", "bigger. Activating this may slow the server down. (Note: this setting", "only affects the web application.) Default is on.").execute(v -> cfg.setQuickContextScan((Boolean) v));
        parser.on("-q", "--quiet", "Run as quietly as possible. Sets logging level to WARNING.").execute(v -> LoggerUtil.setBaseConsoleLogLevel(Level.WARNING));
        parser.on("-R /path/to/configuration", "Read configuration from the specified file.").execute(v -> {
        // Already handled above. This populates usage.
        });
        parser.on("-r", "--remote", "=on|off|uionly|dirbased", REMOTE_REPO_CHOICES, "Specify support for remote SCM systems.", "      on - allow retrieval for remote SCM systems.", "     off - ignore SCM for remote systems.", "  uionly - support remote SCM for user interface only.", "dirbased - allow retrieval during history index only for repositories", "           which allow getting history for directories.").execute(v -> {
            String option = (String) v;
            if (option.equalsIgnoreCase(ON)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.ON);
            } else if (option.equalsIgnoreCase(OFF)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.OFF);
            } else if (option.equalsIgnoreCase(DIRBASED)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.DIRBASED);
            } else if (option.equalsIgnoreCase(UIONLY)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.UIONLY);
            }
        });
        parser.on("--renamedHistory", "=on|off", ON_OFF, Boolean.class, "Enable or disable generating history for renamed files.", "If set to on, makes history indexing slower for repositories", "with lots of renamed files. Default is off.").execute(v -> cfg.setHandleHistoryOfRenamedFiles((Boolean) v));
        parser.on("--repository", "=[path/to/repository|@file_with_paths]", "Path (relative to the source root) to a repository for generating", "history (if -H,--history is on). By default all discovered repositories", "are history-eligible; using --repository limits to only those specified.", "File containing paths can be specified via @path syntax.", "Option may be repeated.").execute(v -> handlePathParameter(repositories, ((String) v).trim()));
        parser.on("-S", "--search", "=[path/to/repository|@file_with_paths]", "Search for source repositories under -s,--source, and add them. Path", "(relative to the source root) is optional. ", "File containing paths can be specified via @path syntax.", "Option may be repeated.").execute(v -> {
            searchRepositories = true;
            String value = ((String) v).trim();
            if (!value.isEmpty()) {
                handlePathParameter(searchPaths, value);
            }
        });
        parser.on("-s", "--source", "=/path/to/source/root", "The root directory of the source tree.").execute(source -> {
            File sourceRoot = new File((String) source);
            if (!sourceRoot.isDirectory()) {
                die("Source root " + sourceRoot + " must be a directory");
            }
            try {
                cfg.setSourceRoot(sourceRoot.getCanonicalPath());
            } catch (IOException e) {
                die(e.getMessage());
            }
        });
        parser.on("--style", "=path", "Path to the subdirectory in the web application containing the requested", "stylesheet. The factory-setting is: \"default\".").execute(stylePath -> cfg.setWebappLAF((String) stylePath));
        parser.on("-T", "--threads", "=number", Integer.class, "The number of threads to use for index generation, repository scan", "and repository invalidation.", "By default the number of threads will be set to the number of available", "CPUs. This influences the number of spawned ctags processes as well.").execute(threadCount -> cfg.setIndexingParallelism((Integer) threadCount));
        parser.on("-t", "--tabSize", "=number", Integer.class, "Default tab size to use (number of spaces per tab character).").execute(tabSize -> cfg.setTabSize((Integer) tabSize));
        parser.on("--token", "=string|@file_with_string", "Authorization bearer API token to use when making API calls", "to the web application").execute(optarg -> {
            String value = ((String) optarg).trim();
            if (value.startsWith("@")) {
                try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(Path.of(value).toString().substring(1))))) {
                    String token = in.readLine().trim();
                    cfg.setIndexerAuthenticationToken(token);
                } catch (IOException e) {
                    die("Failed to read from " + value);
                }
            } else {
                cfg.setIndexerAuthenticationToken(value);
            }
        });
        parser.on("-U", "--uri", "=SCHEME://webappURI:port/contextPath", "Send the current configuration to the specified web application.").execute(webAddr -> {
            webappURI = (String) webAddr;
            try {
                URI uri = new URI(webappURI);
                String scheme = uri.getScheme();
                if (!scheme.equals("http") && !scheme.equals("https")) {
                    die("webappURI '" + webappURI + "' does not have HTTP/HTTPS scheme");
                }
            } catch (URISyntaxException e) {
                die("URL '" + webappURI + "' is not valid.");
            }
            env = RuntimeEnvironment.getInstance();
            env.setConfigURI(webappURI);
        });
        // For unit test only, will not appear in help
        parser.on("---unitTest");
        parser.on("--updateConfig", "Populate the web application with a bare configuration, and exit.").execute(v -> bareConfig = true);
        parser.on("--userPage", "=URL", "Base URL of the user Information provider.", "Example: \"https://www.example.org/viewProfile.jspa?username=\".", "Use \"none\" to disable link.").execute(v -> cfg.setUserPage((String) v));
        parser.on("--userPageSuffix", "=URL-suffix", "URL Suffix for the user Information provider. Default: \"\".").execute(suffix -> cfg.setUserPageSuffix((String) suffix));
        parser.on("-V", "--version", "Print version, and quit.").execute(v -> {
            System.out.println(Info.getFullVersion());
            System.exit(0);
        });
        parser.on("-v", "--verbose", "Set logging level to INFO.").execute(v -> {
            verbose = true;
            LoggerUtil.setBaseConsoleLogLevel(Level.INFO);
        });
        parser.on("-W", "--writeConfig", "=/path/to/configuration", "Write the current configuration to the specified file (so that the web", "application can use the same configuration).").execute(configFile -> configFilename = (String) configFile);
        parser.on("--webappCtags", "=on|off", ON_OFF, Boolean.class, "Web application should run ctags when necessary. Default is off.").execute(v -> cfg.setWebappCtags((Boolean) v));
    });
    // Need to read the configuration file first
    // so that options may be overwritten later.
    configure.parse(argv);
    LOGGER.log(Level.INFO, "Indexer options: {0}", Arrays.toString(argv));
    if (cfg == null) {
        cfg = new Configuration();
    }
    // force user to turn on history capture
    cfg.setHistoryEnabled(false);
    argv = optParser.parse(argv);
    return argv;
}

Example 3 with Repository

use of org.opengrok.indexer.history.Repository in project OpenGrok by OpenGrok.

the class Indexer method main.

/**
 * Program entry point.
 *
 * @param argv argument vector
 */
@SuppressWarnings("PMD.UseStringBufferForStringAppends")
public static void main(String[] argv) {
    // this won't count JVM creation though
    Statistics stats = new Statistics();
    boolean update = true;
    Executor.registerErrorHandler();
    List<String> subFiles = RuntimeEnvironment.getInstance().getSubFiles();
    ArrayList<String> subFilesList = new ArrayList<>();
    boolean createDict = false;
    try {
        argv = parseOptions(argv);
        if (webappURI != null && !HostUtil.isReachable(webappURI, WEBAPP_CONNECT_TIMEOUT)) {
            System.err.println(webappURI + " is not reachable.");
            System.exit(1);
        }
        /*
             * Attend to disabledRepositories here in case exitWithHelp() will
             * need to report about repos.
             */
        disabledRepositories.addAll(cfg.getDisabledRepositories());
        cfg.setDisabledRepositories(disabledRepositories);
        for (String repoName : disabledRepositories) {
            LOGGER.log(Level.FINEST, "Disabled {0}", repoName);
        }
        if (help) {
            exitWithHelp();
        }
        checkConfiguration();
        if (awaitProfiler) {
            pauseToAwaitProfiler();
        }
        env = RuntimeEnvironment.getInstance();
        env.setIndexer(true);
        // Complete the configuration of repository types.
        List<Class<? extends Repository>> repositoryClasses = RepositoryFactory.getRepositoryClasses();
        for (Class<? extends Repository> clazz : repositoryClasses) {
            // Set external repository binaries from System properties.
            try {
                Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
                Object key = f.get(null);
                if (key != null) {
                    cfg.setRepoCmd(clazz.getCanonicalName(), System.getProperty(key.toString()));
                }
            } catch (Exception e) {
            // don't care
            }
        }
        // Logging starts here.
        if (verbose) {
            String fn = LoggerUtil.getFileHandlerPattern();
            if (fn != null) {
                System.out.println("Logging filehandler pattern: " + fn);
            }
        }
        // automatically allow symlinks that are directly in source root
        File sourceRootFile = new File(cfg.getSourceRoot());
        File[] projectDirs = sourceRootFile.listFiles();
        if (projectDirs != null) {
            for (File projectDir : projectDirs) {
                if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
                    allowedSymlinks.add(projectDir.getAbsolutePath());
                }
            }
        }
        allowedSymlinks.addAll(cfg.getAllowedSymlinks());
        cfg.setAllowedSymlinks(allowedSymlinks);
        canonicalRoots.addAll(cfg.getCanonicalRoots());
        cfg.setCanonicalRoots(canonicalRoots);
        // This will be used to perform more fine-grained checking in invalidateRepositories().
        for (String arg : argv) {
            String path = Paths.get(cfg.getSourceRoot(), arg).toString();
            subFilesList.add(path);
        }
        // according to the project key which is the same.
        for (Entry<String, Project> entry : cfg.getProjects().entrySet()) {
            if (entry.getValue().getName() == null) {
                entry.getValue().setName(entry.getKey());
            }
        }
        // with return code upon failure.
        if (checkIndex) {
            if (cfg.getDataRoot() == null || cfg.getDataRoot().isEmpty()) {
                System.err.println("Need data root in configuration for index check (use -R)");
                System.exit(1);
            }
            if (!IndexCheck.check(cfg, subFilesList)) {
                System.err.printf("Index check failed%n");
                System.err.print("You might want to remove " + (!subFilesList.isEmpty() ? "data for projects " + String.join(",", subFilesList) : "all data") + " under the data root and reindex\n");
                System.exit(1);
            }
            System.exit(0);
        }
        // Set updated configuration in RuntimeEnvironment.
        env.setConfiguration(cfg, subFilesList, CommandTimeoutType.INDEXER);
        // Let repository types to add items to ignoredNames.
        // This changes env so is called after the setConfiguration()
        // call above.
        RepositoryFactory.initializeIgnoredNames(env);
        if (bareConfig) {
            getInstance().sendToConfigHost(env, webappURI);
            writeConfigToFile(env, configFilename);
            System.exit(0);
        }
        /*
             * Add paths to directories under source root. If projects
             * are enabled the path should correspond to a project because
             * project path is necessary to correctly set index directory
             * (otherwise the index files will end up in index data root
             * directory and not per project data root directory).
             * For the check we need to have 'env' already set.
             */
        for (String path : subFilesList) {
            String srcPath = env.getSourceRootPath();
            if (srcPath == null) {
                System.err.println("Error getting source root from environment. Exiting.");
                System.exit(1);
            }
            path = path.substring(srcPath.length());
            if (env.hasProjects()) {
                // The paths need to correspond to a project.
                Project project;
                if ((project = Project.getProject(path)) != null) {
                    subFiles.add(path);
                    List<RepositoryInfo> repoList = env.getProjectRepositoriesMap().get(project);
                    if (repoList != null) {
                        repositories.addAll(repoList.stream().map(RepositoryInfo::getDirectoryNameRelative).collect(Collectors.toSet()));
                    }
                } else {
                    System.err.println("The path " + path + " does not correspond to a project");
                }
            } else {
                subFiles.add(path);
            }
        }
        if (!subFilesList.isEmpty() && subFiles.isEmpty()) {
            System.err.println("None of the paths were added, exiting");
            System.exit(1);
        }
        Metrics.updateSubFiles(subFiles);
        // emitted during indexing do not cause validation error.
        if (addProjects && webappURI != null) {
            try {
                IndexerUtil.enableProjects(webappURI);
            } catch (Exception e) {
                LOGGER.log(Level.SEVERE, String.format("Couldn't notify the webapp on %s.", webappURI), e);
                System.err.printf("Couldn't notify the webapp on %s: %s.%n", webappURI, e.getLocalizedMessage());
            }
        }
        LOGGER.log(Level.INFO, "Indexer version {0} ({1}) running on Java {2}", new Object[] { Info.getVersion(), Info.getRevision(), System.getProperty("java.version") });
        // Create history cache first.
        if (searchRepositories) {
            if (searchPaths.isEmpty()) {
                String[] dirs = env.getSourceRootFile().list((f, name) -> f.isDirectory() && env.getPathAccepter().accept(f));
                if (dirs != null) {
                    searchPaths.addAll(Arrays.asList(dirs));
                }
            }
            searchPaths = searchPaths.stream().map(t -> Paths.get(env.getSourceRootPath(), t).toString()).collect(Collectors.toSet());
        }
        getInstance().prepareIndexer(env, searchPaths, addProjects, createDict, runIndex, subFiles, new ArrayList<>(repositories));
        // prepareIndexer() populated the list of projects so now default projects can be set.
        env.setDefaultProjectsFromNames(defaultProjects);
        // And now index it all.
        if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
            IndexChangedListener progress = new DefaultIndexChangedListener();
            getInstance().doIndexerExecution(update, subFiles, progress);
        }
        writeConfigToFile(env, configFilename);
        // or send new configuration to the web application in the case of full reindex.
        if (webappURI != null) {
            if (!subFiles.isEmpty()) {
                getInstance().refreshSearcherManagers(env, subFiles, webappURI);
            } else {
                getInstance().sendToConfigHost(env, webappURI);
            }
        }
        env.getIndexerParallelizer().bounce();
    } catch (ParseException e) {
        System.err.println("** " + e.getMessage());
        System.exit(1);
    } catch (IndexerException ex) {
        LOGGER.log(Level.SEVERE, "Exception running indexer", ex);
        System.err.println("Exception: " + ex.getLocalizedMessage());
        System.err.println(optParser.getUsage());
        System.exit(1);
    } catch (Throwable e) {
        LOGGER.log(Level.SEVERE, "Unexpected Exception", e);
        System.err.println("Exception: " + e.getLocalizedMessage());
        System.exit(1);
    } finally {
        stats.report(LOGGER, "Indexer finished", "indexer.total");
    }
}

Example 4 with Repository

use of org.opengrok.indexer.history.Repository in project OpenGrok by OpenGrok.

the class IndexerTest method testIndexWithSetIndexVersionedFilesOnly.

/**
 * Test indexing w.r.t. setIndexVersionedFilesOnly() setting,
 * i.e. if this option is set to true, index only files tracked by SCM.
 * @throws Exception
 */
@Test
void testIndexWithSetIndexVersionedFilesOnly() throws Exception {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    env.setSourceRoot(repository.getSourceRoot());
    env.setDataRoot(repository.getDataRoot());
    env.setRepositories(repository.getSourceRoot());
    List<RepositoryInfo> repos = env.getRepositories();
    Repository r = null;
    for (RepositoryInfo ri : repos) {
        if (ri.getDirectoryName().equals(repository.getSourceRoot() + "/rfe2575")) {
            r = RepositoryFactory.getRepository(ri, CommandTimeoutType.INDEXER);
            break;
        }
    }
    if (r != null && r.isWorking()) {
        Project project = new Project("rfe2575");
        project.setPath("/rfe2575");
        IndexDatabase idb = new IndexDatabase(project);
        assertNotNull(idb);
        MyIndexChangeListener listener = new MyIndexChangeListener();
        idb.addIndexChangedListener(listener);
        idb.update();
        assertEquals(2, listener.files.size());
        repository.purgeData();
        RuntimeEnvironment.getInstance().setIndexVersionedFilesOnly(true);
        idb = new IndexDatabase(project);
        listener = new MyIndexChangeListener();
        idb.addIndexChangedListener(listener);
        idb.update();
        assertEquals(1, listener.files.size());
        RuntimeEnvironment.getInstance().setIndexVersionedFilesOnly(false);
    } else {
        System.out.println("Skipping test. Repository for rfe2575 not found or an sccs I could use in path.");
    }
}

Example 5 with Repository

use of org.opengrok.indexer.history.Repository in project OpenGrok by OpenGrok.

the class ProjectsController method markIndexed.

@PUT
@Path("/{project}/indexed")
@Consumes(MediaType.TEXT_PLAIN)
public Response markIndexed(@Context HttpServletRequest request, @PathParam("project") String projectNameParam) {
    // Avoid classification as a taint bug.
    final String projectName = Laundromat.launderInput(projectNameParam);
    Project project = env.getProjects().get(projectName);
    if (project == null) {
        LOGGER.log(Level.WARNING, "cannot find project {0} to mark as indexed", projectName);
        throw new NotFoundException(String.format("project '%s' does not exist", projectName));
    }
    project.setIndexed(true);
    return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, new ApiTask(request.getRequestURI(), () -> {
        // Refresh current version of the project's repositories.
        List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project);
        if (riList != null) {
            for (RepositoryInfo ri : riList) {
                Repository repo = getRepository(ri, CommandTimeoutType.RESTFUL);
                if (repo != null && repo.getCurrentVersion() != null && repo.getCurrentVersion().length() > 0) {
                    // getRepository() always creates fresh instance
                    // of the Repository object so there is no need
                    // to call setCurrentVersion() on it.
                    ri.setCurrentVersion(repo.determineCurrentVersion());
                }
            }
        }
        CompletableFuture.runAsync(() -> suggester.rebuild(projectName));
        // In case this project has just been incrementally indexed,
        // its IndexSearcher needs a poke.
        env.maybeRefreshIndexSearchers(Collections.singleton(projectName));
        env.refreshDateForLastIndexRun();
        return null;
    }));
}