Examples with Ctags org.opengrok.indexer.analysis.Ctags used on opensource projects

Example 1 with Ctags

use of org.opengrok.indexer.analysis.Ctags in project OpenGrok by OpenGrok.

the class CSharpAnalyzerFactoryTest method setUpClass.

@BeforeAll
public static void setUpClass() throws Exception {
    ctags = new Ctags();
    repository = new TestRepository();
    repository.create(CSharpAnalyzerFactoryTest.class.getClassLoader().getResource("sources"));
    CSharpAnalyzerFactory analFact = new CSharpAnalyzerFactory();
    analyzer = analFact.getAnalyzer();
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.validateUniversalCtags()) {
        analyzer.setCtags(new Ctags());
    }
}

Example 2 with Ctags

use of org.opengrok.indexer.analysis.Ctags in project OpenGrok by OpenGrok.

the class PascalAnalyzerFactoryTest method setUpClass.

@BeforeAll
public static void setUpClass() throws Exception {
    ctags = new Ctags();
    repository = new TestRepository();
    repository.create(PascalAnalyzerFactoryTest.class.getClassLoader().getResource("sources"));
    PascalAnalyzerFactory analyzerFactory = new PascalAnalyzerFactory();
    analyzer = analyzerFactory.getAnalyzer();
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.validateUniversalCtags()) {
        analyzer.setCtags(new Ctags());
    }
}

Example 3 with Ctags

use of org.opengrok.indexer.analysis.Ctags in project OpenGrok by OpenGrok.

the class CtagsUtil method newInstance.

/**
 * Creates a new instance, and attempts to configure it from the
 * environment.
 * @return a defined instance
 */
public static Ctags newInstance(RuntimeEnvironment env) {
    Ctags ctags = new Ctags();
    ctags.setLangMap(AnalyzerGuru.getLangMap());
    String filename = env.getCTagsExtraOptionsFile();
    if (filename != null) {
        ctags.setCTagsExtraOptionsFile(filename);
    }
    return ctags;
}

Example 4 with Ctags

use of org.opengrok.indexer.analysis.Ctags in project OpenGrok by OpenGrok.

the class Indexer method parseOptions.

/**
 * Parse OpenGrok Indexer options
 * This method was created so that it would be easier to write unit
 * tests against the Indexer option parsing mechanism.
 *
 * @param argv the command line arguments
 * @return array of remaining non option arguments
 * @throws ParseException if parsing failed
 */
public static String[] parseOptions(String[] argv) throws ParseException {
    final String[] usage = { HELP_OPT_1 };
    if (argv.length == 0) {
        // will force usage output
        argv = usage;
        // with non-zero EXIT STATUS
        status = 1;
    }
    /*
         * Pre-match any of the --help options so that some possible exception-generating args handlers (e.g. -R)
         * can be short-circuited.
         */
    boolean preHelp = Arrays.stream(argv).anyMatch(s -> HELP_OPT_1.equals(s) || HELP_OPT_2.equals(s) || HELP_OPT_3.equals(s));
    OptionParser configure = OptionParser.scan(parser -> parser.on("-R configPath").execute(cfgFile -> {
        try {
            cfg = Configuration.read(new File((String) cfgFile));
        } catch (IOException e) {
            if (!preHelp) {
                die(e.getMessage());
            } else {
                System.err.printf("Warning: failed to read -R %s%n", cfgFile);
            }
        }
    }));
    searchPaths.clear();
    // Limit usage lines to 72 characters for concise formatting.
    optParser = OptionParser.execute(parser -> {
        parser.setPrologue(String.format("%nUsage: java -jar %s [options] [subDir1 [...]]%n", OPENGROK_JAR));
        parser.on(HELP_OPT_3, HELP_OPT_2, HELP_OPT_1, "=[mode]", "With no mode specified, display this usage summary. Or specify a mode:", "  config - display configuration.xml examples.", "   ctags - display ctags command-line.", "    guru - display AnalyzerGuru details.", "   repos - display enabled repositories.").execute(v -> {
            help = true;
            helpUsage = parser.getUsage();
            String mode = (String) v;
            if (mode != null && !mode.isEmpty()) {
                try {
                    helpMode = HelpMode.valueOf(((String) v).toUpperCase(Locale.ROOT));
                } catch (IllegalArgumentException ex) {
                    die("mode '" + v + "' is not valid.");
                }
            }
        });
        parser.on("--apiTimeout", "=number", Integer.class, "Set timeout for asynchronous API requests.").execute(v -> cfg.setApiTimeout((Integer) v));
        parser.on("--connectTimeout", "=number", Integer.class, "Set connect timeout. Used for API requests.").execute(v -> cfg.setConnectTimeout((Integer) v));
        parser.on("-A (.ext|prefix.):(-|analyzer)", "--analyzer", "/(\\.\\w+|\\w+\\.):(-|[a-zA-Z_0-9.]+)/", "Associates files with the specified prefix or extension (case-", "insensitive) to be analyzed with the given analyzer, where 'analyzer'", "may be specified using a class name (case-sensitive e.g. RubyAnalyzer)", "or analyzer language name (case-sensitive e.g. C). Option may be", "repeated.", "  Ex: -A .foo:CAnalyzer", "      will use the C analyzer for all files ending with .FOO", "  Ex: -A bar.:Perl", "      will use the Perl analyzer for all files starting with", "      \"BAR\" (no full-stop)", "  Ex: -A .c:-", "      will disable specialized analyzers for all files ending with .c").execute(analyzerSpec -> {
            String[] arg = ((String) analyzerSpec).split(":");
            String fileSpec = arg[0];
            String analyzer = arg[1];
            configureFileAnalyzer(fileSpec, analyzer);
        });
        parser.on("-c", "--ctags", "=/path/to/ctags", "Path to Universal Ctags. Default is ctags in environment PATH.").execute(v -> cfg.setCtags((String) v));
        parser.on("--canonicalRoot", "=/path/", "Allow symlinks to canonical targets starting with the specified root", "without otherwise needing to specify -N,--symlink for such symlinks. A", "canonical root must end with a file separator. For security, a canonical", "root cannot be the root directory. Option may be repeated.").execute(v -> {
            String root = (String) v;
            String problem = CanonicalRootValidator.validate(root, "--canonicalRoot");
            if (problem != null) {
                die(problem);
            }
            canonicalRoots.add(root);
        });
        parser.on("--checkIndex", "Check index, exit with 0 on success,", "with 1 on failure.").execute(v -> checkIndex = true);
        parser.on("-d", "--dataRoot", "=/path/to/data/root", "The directory where OpenGrok stores the generated data.").execute(drPath -> {
            File dataRoot = new File((String) drPath);
            if (!dataRoot.exists() && !dataRoot.mkdirs()) {
                die("Cannot create data root: " + dataRoot);
            }
            if (!dataRoot.isDirectory()) {
                die("Data root must be a directory");
            }
            try {
                cfg.setDataRoot(dataRoot.getCanonicalPath());
            } catch (IOException e) {
                die(e.getMessage());
            }
        });
        parser.on("--depth", "=number", Integer.class, "Scanning depth for repositories in directory structure relative to", "source root. Default is " + Configuration.defaultScanningDepth + ".").execute(depth -> cfg.setScanningDepth((Integer) depth));
        parser.on("--disableRepository", "=type_name", "Disables operation of an OpenGrok-supported repository. See also", "-h,--help repos. Option may be repeated.", "  Ex: --disableRepository git", "      will disable the GitRepository", "  Ex: --disableRepository MercurialRepository").execute(v -> {
            String repoType = (String) v;
            String repoSimpleType = RepositoryFactory.matchRepositoryByName(repoType);
            if (repoSimpleType == null) {
                System.err.printf("'--disableRepository %s' does not match a type and is ignored%n", v);
            } else {
                disabledRepositories.add(repoSimpleType);
            }
        });
        parser.on("-e", "--economical", "To consume less disk space, OpenGrok will not generate and save", "hypertext cross-reference files but will generate on demand, which could", "be slightly slow.").execute(v -> cfg.setGenerateHtml(false));
        parser.on("-G", "--assignTags", "Assign commit tags to all entries in history for all repositories.").execute(v -> cfg.setTagsEnabled(true));
        parser.on("-H", "--history", "Enable history.").execute(v -> cfg.setHistoryEnabled(true));
        parser.on("--historyThreads", "=number", Integer.class, "The number of threads to use for history cache generation on repository level. " + "By default the number of threads will be set to the number of available CPUs.", "Assumes -H/--history.").execute(threadCount -> cfg.setHistoryParallelism((Integer) threadCount));
        parser.on("--historyFileThreads", "=number", Integer.class, "The number of threads to use for history cache generation when dealing with individual files.", "By default the number of threads will be set to the number of available CPUs.", "Assumes -H/--history.").execute(threadCount -> cfg.setHistoryFileParallelism((Integer) threadCount));
        parser.on("-I", "--include", "=pattern", "Only files matching this pattern will be examined. Pattern supports", "wildcards (example: -I '*.java' -I '*.c'). Option may be repeated.").execute(pattern -> cfg.getIncludedNames().add((String) pattern));
        parser.on("-i", "--ignore", "=pattern", "Ignore matching files (prefixed with 'f:' or no prefix) or directories", "(prefixed with 'd:'). Pattern supports wildcards (example: -i '*.so'", "-i d:'test*'). Option may be repeated.").execute(pattern -> cfg.getIgnoredNames().add((String) pattern));
        parser.on("-l", "--lock", "=on|off|simple|native", LUCENE_LOCKS, "Set OpenGrok/Lucene locking mode of the Lucene database during index", "generation. \"on\" is an alias for \"simple\". Default is off.").execute(v -> {
            try {
                if (v != null) {
                    String vuc = v.toString().toUpperCase(Locale.ROOT);
                    cfg.setLuceneLocking(LuceneLockName.valueOf(vuc));
                }
            } catch (IllegalArgumentException e) {
                System.err.printf("`--lock %s' is invalid and ignored%n", v);
            }
        });
        parser.on("--leadingWildCards", "=on|off", ON_OFF, Boolean.class, "Allow or disallow leading wildcards in a search. Default is on.").execute(v -> cfg.setAllowLeadingWildcard((Boolean) v));
        parser.on("-m", "--memory", "=number", Double.class, "Amount of memory (MB) that may be used for buffering added documents and", "deletions before they are flushed to the directory (default " + Configuration.defaultRamBufferSize + ").", "Please increase JVM heap accordingly too.").execute(memSize -> cfg.setRamBufferSize((Double) memSize));
        parser.on("--mandoc", "=/path/to/mandoc", "Path to mandoc(1) binary.").execute(mandocPath -> cfg.setMandoc((String) mandocPath));
        parser.on("-N", "--symlink", "=/path/to/symlink", "Allow the symlink to be followed. Other symlinks targeting the same", "canonical target or canonical children will be allowed too. Option may", "be repeated. (By default only symlinks directly under the source root", "directory are allowed. See also --canonicalRoot)").execute(v -> allowedSymlinks.add((String) v));
        parser.on("-n", "--noIndex", "Do not generate indexes and other data (such as history cache and xref", "files), but process all other command line options.").execute(v -> runIndex = false);
        parser.on("--nestingMaximum", "=number", Integer.class, "Maximum depth of nested repositories. Default is 1.").execute(v -> cfg.setNestingMaximum((Integer) v));
        parser.on("-O", "--optimize", "=on|off", ON_OFF, Boolean.class, "Turn on/off the optimization of the index database as part of the", "indexing step. Default is on.").execute(v -> {
            boolean oldval = cfg.isOptimizeDatabase();
            cfg.setOptimizeDatabase((Boolean) v);
            if (oldval != cfg.isOptimizeDatabase()) {
                optimizedChanged = true;
            }
        });
        parser.on("-o", "--ctagOpts", "=path", "File with extra command line options for ctags.").execute(path -> {
            String CTagsExtraOptionsFile = (String) path;
            File CTagsFile = new File(CTagsExtraOptionsFile);
            if (!(CTagsFile.isFile() && CTagsFile.canRead())) {
                die("File '" + CTagsExtraOptionsFile + "' not found for the -o option");
            }
            System.err.println("INFO: file with extra " + "options for ctags: " + CTagsExtraOptionsFile);
            cfg.setCTagsExtraOptionsFile(CTagsExtraOptionsFile);
        });
        parser.on("-P", "--projects", "Generate a project for each top-level directory in source root.").execute(v -> {
            addProjects = true;
            cfg.setProjectsEnabled(true);
        });
        parser.on("-p", "--defaultProject", "=path/to/default/project", "Path (relative to the source root) to a project that should be selected", "by default in the web application (when no other project is set either", "in a cookie or in parameter). Option may be repeated to specify several", "projects. Use the special value __all__ to indicate all projects.").execute(v -> defaultProjects.add((String) v));
        parser.on("--profiler", "Pause to await profiler or debugger.").execute(v -> awaitProfiler = true);
        parser.on("--progress", "Print per-project percentage progress information.").execute(v -> cfg.setPrintProgress(true));
        parser.on("-Q", "--quickScan", "=on|off", ON_OFF, Boolean.class, "Turn on/off quick context scan. By default, only the first 1024KB of a", "file is scanned, and a link ('[..all..]') is inserted when the file is", "bigger. Activating this may slow the server down. (Note: this setting", "only affects the web application.) Default is on.").execute(v -> cfg.setQuickContextScan((Boolean) v));
        parser.on("-q", "--quiet", "Run as quietly as possible. Sets logging level to WARNING.").execute(v -> LoggerUtil.setBaseConsoleLogLevel(Level.WARNING));
        parser.on("-R /path/to/configuration", "Read configuration from the specified file.").execute(v -> {
        // Already handled above. This populates usage.
        });
        parser.on("-r", "--remote", "=on|off|uionly|dirbased", REMOTE_REPO_CHOICES, "Specify support for remote SCM systems.", "      on - allow retrieval for remote SCM systems.", "     off - ignore SCM for remote systems.", "  uionly - support remote SCM for user interface only.", "dirbased - allow retrieval during history index only for repositories", "           which allow getting history for directories.").execute(v -> {
            String option = (String) v;
            if (option.equalsIgnoreCase(ON)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.ON);
            } else if (option.equalsIgnoreCase(OFF)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.OFF);
            } else if (option.equalsIgnoreCase(DIRBASED)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.DIRBASED);
            } else if (option.equalsIgnoreCase(UIONLY)) {
                cfg.setRemoteScmSupported(Configuration.RemoteSCM.UIONLY);
            }
        });
        parser.on("--renamedHistory", "=on|off", ON_OFF, Boolean.class, "Enable or disable generating history for renamed files.", "If set to on, makes history indexing slower for repositories", "with lots of renamed files. Default is off.").execute(v -> cfg.setHandleHistoryOfRenamedFiles((Boolean) v));
        parser.on("--repository", "=[path/to/repository|@file_with_paths]", "Path (relative to the source root) to a repository for generating", "history (if -H,--history is on). By default all discovered repositories", "are history-eligible; using --repository limits to only those specified.", "File containing paths can be specified via @path syntax.", "Option may be repeated.").execute(v -> handlePathParameter(repositories, ((String) v).trim()));
        parser.on("-S", "--search", "=[path/to/repository|@file_with_paths]", "Search for source repositories under -s,--source, and add them. Path", "(relative to the source root) is optional. ", "File containing paths can be specified via @path syntax.", "Option may be repeated.").execute(v -> {
            searchRepositories = true;
            String value = ((String) v).trim();
            if (!value.isEmpty()) {
                handlePathParameter(searchPaths, value);
            }
        });
        parser.on("-s", "--source", "=/path/to/source/root", "The root directory of the source tree.").execute(source -> {
            File sourceRoot = new File((String) source);
            if (!sourceRoot.isDirectory()) {
                die("Source root " + sourceRoot + " must be a directory");
            }
            try {
                cfg.setSourceRoot(sourceRoot.getCanonicalPath());
            } catch (IOException e) {
                die(e.getMessage());
            }
        });
        parser.on("--style", "=path", "Path to the subdirectory in the web application containing the requested", "stylesheet. The factory-setting is: \"default\".").execute(stylePath -> cfg.setWebappLAF((String) stylePath));
        parser.on("-T", "--threads", "=number", Integer.class, "The number of threads to use for index generation, repository scan", "and repository invalidation.", "By default the number of threads will be set to the number of available", "CPUs. This influences the number of spawned ctags processes as well.").execute(threadCount -> cfg.setIndexingParallelism((Integer) threadCount));
        parser.on("-t", "--tabSize", "=number", Integer.class, "Default tab size to use (number of spaces per tab character).").execute(tabSize -> cfg.setTabSize((Integer) tabSize));
        parser.on("--token", "=string|@file_with_string", "Authorization bearer API token to use when making API calls", "to the web application").execute(optarg -> {
            String value = ((String) optarg).trim();
            if (value.startsWith("@")) {
                try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(Path.of(value).toString().substring(1))))) {
                    String token = in.readLine().trim();
                    cfg.setIndexerAuthenticationToken(token);
                } catch (IOException e) {
                    die("Failed to read from " + value);
                }
            } else {
                cfg.setIndexerAuthenticationToken(value);
            }
        });
        parser.on("-U", "--uri", "=SCHEME://webappURI:port/contextPath", "Send the current configuration to the specified web application.").execute(webAddr -> {
            webappURI = (String) webAddr;
            try {
                URI uri = new URI(webappURI);
                String scheme = uri.getScheme();
                if (!scheme.equals("http") && !scheme.equals("https")) {
                    die("webappURI '" + webappURI + "' does not have HTTP/HTTPS scheme");
                }
            } catch (URISyntaxException e) {
                die("URL '" + webappURI + "' is not valid.");
            }
            env = RuntimeEnvironment.getInstance();
            env.setConfigURI(webappURI);
        });
        // For unit test only, will not appear in help
        parser.on("---unitTest");
        parser.on("--updateConfig", "Populate the web application with a bare configuration, and exit.").execute(v -> bareConfig = true);
        parser.on("--userPage", "=URL", "Base URL of the user Information provider.", "Example: \"https://www.example.org/viewProfile.jspa?username=\".", "Use \"none\" to disable link.").execute(v -> cfg.setUserPage((String) v));
        parser.on("--userPageSuffix", "=URL-suffix", "URL Suffix for the user Information provider. Default: \"\".").execute(suffix -> cfg.setUserPageSuffix((String) suffix));
        parser.on("-V", "--version", "Print version, and quit.").execute(v -> {
            System.out.println(Info.getFullVersion());
            System.exit(0);
        });
        parser.on("-v", "--verbose", "Set logging level to INFO.").execute(v -> {
            verbose = true;
            LoggerUtil.setBaseConsoleLogLevel(Level.INFO);
        });
        parser.on("-W", "--writeConfig", "=/path/to/configuration", "Write the current configuration to the specified file (so that the web", "application can use the same configuration).").execute(configFile -> configFilename = (String) configFile);
        parser.on("--webappCtags", "=on|off", ON_OFF, Boolean.class, "Web application should run ctags when necessary. Default is off.").execute(v -> cfg.setWebappCtags((Boolean) v));
    });
    // Need to read the configuration file first
    // so that options may be overwritten later.
    configure.parse(argv);
    LOGGER.log(Level.INFO, "Indexer options: {0}", Arrays.toString(argv));
    if (cfg == null) {
        cfg = new Configuration();
    }
    // force user to turn on history capture
    cfg.setHistoryEnabled(false);
    argv = optParser.parse(argv);
    return argv;
}

Example 5 with Ctags

use of org.opengrok.indexer.analysis.Ctags in project OpenGrok by OpenGrok.

the class IndexDatabase method indexParallel.

/**
 * Executes the second, parallel stage of indexing.
 * @param dir the parent directory (when appended to SOURCE_ROOT)
 * @param args contains a list of files to index, found during the earlier
 * stage
 */
private void indexParallel(String dir, IndexDownArgs args) {
    int worksCount = args.works.size();
    if (worksCount < 1) {
        return;
    }
    AtomicInteger successCounter = new AtomicInteger();
    AtomicInteger currentCounter = new AtomicInteger();
    AtomicInteger alreadyClosedCounter = new AtomicInteger();
    IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer();
    ObjectPool<Ctags> ctagsPool = parallelizer.getCtagsPool();
    Map<Boolean, List<IndexFileWork>> bySuccess = null;
    try (Progress progress = new Progress(LOGGER, dir, worksCount)) {
        bySuccess = parallelizer.getForkJoinPool().submit(() -> args.works.parallelStream().collect(Collectors.groupingByConcurrent((x) -> {
            int tries = 0;
            Ctags pctags = null;
            boolean ret;
            Statistics stats = new Statistics();
            while (true) {
                try {
                    if (alreadyClosedCounter.get() > 0) {
                        ret = false;
                    } else {
                        pctags = ctagsPool.get();
                        addFile(x.file, x.path, pctags);
                        successCounter.incrementAndGet();
                        ret = true;
                    }
                } catch (AlreadyClosedException e) {
                    alreadyClosedCounter.incrementAndGet();
                    String errmsg = String.format("ERROR addFile(): %s", x.file);
                    LOGGER.log(Level.SEVERE, errmsg, e);
                    x.exception = e;
                    ret = false;
                } catch (InterruptedException e) {
                    // Allow one retry if interrupted
                    if (++tries <= 1) {
                        continue;
                    }
                    LOGGER.log(Level.WARNING, "No retry: {0}", x.file);
                    x.exception = e;
                    ret = false;
                } catch (RuntimeException | IOException e) {
                    String errmsg = String.format("ERROR addFile(): %s", x.file);
                    LOGGER.log(Level.WARNING, errmsg, e);
                    x.exception = e;
                    ret = false;
                } finally {
                    if (pctags != null) {
                        pctags.reset();
                        ctagsPool.release(pctags);
                    }
                }
                progress.increment();
                stats.report(LOGGER, Level.FINEST, String.format("file ''%s'' %s", x.file, ret ? "indexed" : "failed indexing"));
                return ret;
            }
        }))).get();
    } catch (InterruptedException | ExecutionException e) {
        int successCount = successCounter.intValue();
        double successPct = 100.0 * successCount / worksCount;
        String exmsg = String.format("%d successes (%.1f%%) after aborting parallel-indexing", successCount, successPct);
        LOGGER.log(Level.SEVERE, exmsg, e);
    }
    args.cur_count = currentCounter.intValue();
    // Start with failureCount=worksCount, and then subtract successes.
    int failureCount = worksCount;
    if (bySuccess != null) {
        List<IndexFileWork> successes = bySuccess.getOrDefault(Boolean.TRUE, null);
        if (successes != null) {
            failureCount -= successes.size();
        }
    }
    if (failureCount > 0) {
        double pctFailed = 100.0 * failureCount / worksCount;
        String exmsg = String.format("%d failures (%.1f%%) while parallel-indexing", failureCount, pctFailed);
        LOGGER.log(Level.WARNING, exmsg);
    }
    /*
         * Encountering an AlreadyClosedException is severe enough to abort the
         * run, since it will fail anyway later upon trying to commit().
         */
    int numAlreadyClosed = alreadyClosedCounter.get();
    if (numAlreadyClosed > 0) {
        throw new AlreadyClosedException(String.format("count=%d", numAlreadyClosed));
    }
}