use of org.opensolaris.opengrok.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.
the class IndexDatabase method update.
/**
* Update the content of this index database
*
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running!");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
}
if (ctags == null) {
LOGGER.severe("Unable to run ctags! searching definitions will not work!");
}
if (ctags != null) {
String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
if (filename != null) {
ctags.setCTagsExtraOptionsFile(filename);
}
}
try {
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
iwc.setRAMBufferSizeMB(RuntimeEnvironment.getInstance().getRamBufferSize());
writer = new IndexWriter(indexDirectory, iwc);
// to make sure index exists on the disk
writer.commit();
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
// open existing index
IndexReader reader = DirectoryReader.open(indexDirectory);
Terms terms = null;
int numDocs = reader.numDocs();
if (numDocs > 0) {
//reader.getTermVectors(0);
Fields uFields = MultiFields.getFields(reader);
terms = uFields.terms(QueryBuilder.U);
}
try {
if (numDocs > 0) {
uidIter = terms.iterator();
//init uid
TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid));
if (stat == TermsEnum.SeekStatus.END) {
uidIter = null;
LOGGER.log(Level.WARNING, "Couldn't find a start term for {0}, empty u field?", startuid);
}
}
// The code below traverses the tree to get total count.
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
LOGGER.log(Level.INFO, "Counting files in {0} ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
LOGGER.log(Level.INFO, "Need to process: {0} files for {1}", new Object[] { file_cnt, dir });
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter != null && uidIter.term() != null && uidIter.term().utf8ToString().startsWith(startuid)) {
removeFile();
BytesRef next = uidIter.next();
if (next == null) {
uidIter = null;
}
}
} finally {
reader.close();
}
}
} finally {
if (writer != null) {
try {
writer.prepareCommit();
writer.commit();
writer.close();
} catch (IOException e) {
LOGGER.log(Level.WARNING, "An error occured while closing writer", e);
}
}
if (ctags != null) {
try {
ctags.close();
} catch (IOException e) {
LOGGER.log(Level.WARNING, "An error occured while closing ctags process", e);
}
}
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
String purpose = "used for timestamping the index database.";
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
LOGGER.log(Level.WARNING, "Failed to set last modified time on ''{0}'', {1}", new Object[] { timestamp.getAbsolutePath(), purpose });
}
} else {
if (!timestamp.createNewFile()) {
LOGGER.log(Level.WARNING, "Failed to create file ''{0}'', {1}", new Object[] { timestamp.getAbsolutePath(), purpose });
}
}
}
}
use of org.opensolaris.opengrok.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.
the class IndexDatabase method getDefinitions.
/**
* Get the latest definitions for a file from the index.
*
* @param file the file whose definitions to find
* @return definitions for the file, or {@code null} if they could not be
* found
* @throws IOException if an error happens when accessing the index
* @throws ParseException if an error happens when building the Lucene query
* @throws ClassNotFoundException if the class for the stored definitions
* instance cannot be found
*/
public static Definitions getDefinitions(File file) throws IOException, ParseException, ClassNotFoundException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String path = env.getPathRelativeToSourceRoot(file, 0);
//sanitize windows path delimiters
//in order not to conflict with Lucene escape character
path = path.replace("\\", "/");
IndexReader ireader = getIndexReader(path);
if (ireader == null) {
// No index, no definitions...
return null;
}
try {
Query q = new QueryBuilder().setPath(path).build();
IndexSearcher searcher = new IndexSearcher(ireader);
TopDocs top = searcher.search(q, 1);
if (top.totalHits == 0) {
// No hits, no definitions...
return null;
}
Document doc = searcher.doc(top.scoreDocs[0].doc);
String foundPath = doc.get(QueryBuilder.PATH);
// Only use the definitions if we found an exact match.
if (path.equals(foundPath)) {
IndexableField tags = doc.getField(QueryBuilder.TAGS);
if (tags != null) {
return Definitions.deserialize(tags.binaryValue().bytes);
}
}
} finally {
ireader.close();
}
// Didn't find any definitions.
return null;
}
use of org.opensolaris.opengrok.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.
the class IndexDatabase method getIndexReader.
/**
* Get an indexReader for the Index database where a given file
*
* @param path the file to get the database for
* @return The index database where the file should be located or null if it
* cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
FSDirectory fdir = FSDirectory.open(indexDir.toPath(), NoLockFactory.INSTANCE);
if (indexDir.exists() && DirectoryReader.indexExists(fdir)) {
ret = DirectoryReader.open(fdir);
}
} catch (Exception ex) {
LOGGER.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath());
LOGGER.log(Level.FINE, "Stack Trace: ", ex);
}
return ret;
}
use of org.opensolaris.opengrok.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.
the class Indexer method main.
/**
* Program entry point
*
* @param argv argument vector
*/
@SuppressWarnings("PMD.UseStringBufferForStringAppends")
public static void main(String[] argv) {
//this won't count JVM creation though
Statistics stats = new Statistics();
boolean runIndex = true;
boolean update = true;
boolean optimizedChanged = false;
ArrayList<String> zapCache = new ArrayList<>();
CommandLineOptions cmdOptions = new CommandLineOptions();
if (argv.length == 0) {
System.err.println(cmdOptions.getUsage());
System.exit(1);
} else {
Executor.registerErrorHandler();
boolean searchRepositories = false;
ArrayList<String> subFiles = new ArrayList<>();
ArrayList<String> subFilesList = new ArrayList<>();
ArrayList<String> repositories = new ArrayList<>();
HashSet<String> allowedSymlinks = new HashSet<>();
String configFilename = null;
String configHost = null;
boolean addProjects = false;
boolean refreshHistory = false;
String defaultProject = null;
boolean listFiles = false;
boolean listRepos = false;
boolean createDict = false;
int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors());
String host = null;
int port = 0;
// Parse command line options:
Getopt getopt = new Getopt(argv, cmdOptions.getCommandString());
try {
getopt.parse();
} catch (ParseException ex) {
System.err.println("OpenGrok: " + ex.getMessage());
System.err.println(cmdOptions.getUsage());
System.exit(1);
}
try {
Configuration cfg = null;
int cmd;
// will try to overwrite options..
while ((cmd = getopt.getOpt()) != -1) {
if (cmd == 'R') {
cfg = Configuration.read(new File(getopt.getOptarg()));
break;
}
}
if (cfg == null) {
cfg = new Configuration();
}
// Now we can handle all the other options..
getopt.reset();
while ((cmd = getopt.getOpt()) != -1) {
switch(cmd) {
case 'A':
{
String[] arg = getopt.getOptarg().split(":");
boolean prefix = false;
if (arg.length != 2) {
A_usage();
}
if (arg[0].endsWith(".")) {
arg[0] = arg[0].substring(0, arg[0].lastIndexOf('.')).toUpperCase();
prefix = true;
} else if (arg[0].startsWith(".")) {
arg[0] = arg[0].substring(arg[0].lastIndexOf('.') + 1).toUpperCase();
} else {
A_usage();
}
if (arg[1].equals("-")) {
if (prefix) {
AnalyzerGuru.addPrefix(arg[0], null);
} else {
AnalyzerGuru.addExtension(arg[0], null);
}
break;
}
if (prefix) {
try {
AnalyzerGuru.addPrefix(arg[0], AnalyzerGuru.findFactory(arg[1]));
} catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
LOGGER.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
LOGGER.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
System.exit(1);
}
} else {
try {
AnalyzerGuru.addExtension(arg[0], AnalyzerGuru.findFactory(arg[1]));
} catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
LOGGER.log(Level.SEVERE, "Unable to use {0} as a FileAnalyzerFactory", arg[1]);
LOGGER.log(Level.SEVERE, "Stack: ", e.fillInStackTrace());
System.exit(1);
}
}
}
break;
case 'a':
if (getopt.getOptarg().equalsIgnoreCase(ON)) {
cfg.setAllowLeadingWildcard(true);
} else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cfg.setAllowLeadingWildcard(false);
} else {
System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -a");
System.err.println(" Ex: \"-a on\" will allow a search to start with a wildcard");
System.err.println(" \"-a off\" will disallow a search to start with a wildcard");
System.exit(1);
}
break;
case 'B':
cfg.setUserPage(getopt.getOptarg());
break;
case 'C':
cfg.setPrintProgress(true);
break;
case 'c':
cfg.setCtags(getopt.getOptarg());
break;
case 'd':
{
File dataRoot = new File(getopt.getOptarg());
if (!dataRoot.exists() && !dataRoot.mkdirs()) {
System.err.println("ERROR: Cannot create data root");
System.exit(1);
}
if (!dataRoot.isDirectory()) {
System.err.println("ERROR: Data root must be a directory");
System.exit(1);
}
cfg.setDataRoot(dataRoot.getCanonicalPath());
break;
}
case 'e':
cfg.setGenerateHtml(false);
break;
case 'G':
cfg.setTagsEnabled(true);
break;
case 'H':
refreshHistory = true;
break;
case 'h':
repositories.add(getopt.getOptarg());
break;
case 'I':
cfg.getIncludedNames().add(getopt.getOptarg());
break;
case 'i':
cfg.getIgnoredNames().add(getopt.getOptarg());
break;
case 'K':
listRepos = true;
break;
case 'k':
zapCache.add(getopt.getOptarg());
break;
case 'L':
cfg.setWebappLAF(getopt.getOptarg());
break;
case 'l':
if (getopt.getOptarg().equalsIgnoreCase(ON)) {
cfg.setUsingLuceneLocking(true);
} else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cfg.setUsingLuceneLocking(false);
} else {
System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -l");
System.err.println(" Ex: \"-l on\" will enable locks in Lucene");
System.err.println(" \"-l off\" will disable locks in Lucene");
}
break;
case 'm':
{
try {
cfg.setRamBufferSize(Double.parseDouble(getopt.getOptarg()));
} catch (NumberFormatException exp) {
System.err.println("ERROR: Failed to parse argument to \"-m\": " + exp.getMessage());
System.exit(1);
}
break;
}
case 'N':
allowedSymlinks.add(getopt.getOptarg());
break;
case 'n':
runIndex = false;
break;
case 'O':
{
boolean oldval = cfg.isOptimizeDatabase();
if (getopt.getOptarg().equalsIgnoreCase(ON)) {
cfg.setOptimizeDatabase(true);
} else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cfg.setOptimizeDatabase(false);
} else {
System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -O");
System.err.println(" Ex: \"-O on\" will optimize the database as part of the index generation");
System.err.println(" \"-O off\" disable optimization of the index database");
}
if (oldval != cfg.isOptimizeDatabase()) {
optimizedChanged = true;
}
break;
}
case 'o':
String CTagsExtraOptionsFile = getopt.getOptarg();
File CTagsFile = new File(CTagsExtraOptionsFile);
if (!(CTagsFile.isFile() && CTagsFile.canRead())) {
System.err.println("ERROR: File '" + CTagsExtraOptionsFile + "' not found for the -o option");
System.exit(1);
}
System.err.println("INFO: file with extra " + "options for ctags: " + CTagsExtraOptionsFile);
cfg.setCTagsExtraOptionsFile(CTagsExtraOptionsFile);
break;
case 'P':
addProjects = true;
break;
case 'p':
defaultProject = getopt.getOptarg();
break;
case 'Q':
if (getopt.getOptarg().equalsIgnoreCase(ON)) {
cfg.setQuickContextScan(true);
} else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cfg.setQuickContextScan(false);
} else {
System.err.println("ERROR: You should pass either \"on\" or \"off\" as argument to -Q");
System.err.println(" Ex: \"-Q on\" will just scan a \"chunk\" of the file and insert \"[..all..]\"");
System.err.println(" \"-Q off\" will try to build a more accurate list by reading the complete file.");
}
break;
case 'q':
cfg.setVerbose(false);
LoggerUtil.setBaseConsoleLogLevel(Level.WARNING);
break;
case 'R':
// already handled
break;
case 'r':
if (getopt.getOptarg().equalsIgnoreCase(ON)) {
cfg.setRemoteScmSupported(Configuration.RemoteSCM.ON);
} else if (getopt.getOptarg().equalsIgnoreCase(OFF)) {
cfg.setRemoteScmSupported(Configuration.RemoteSCM.OFF);
} else if (getopt.getOptarg().equalsIgnoreCase(DIRBASED)) {
cfg.setRemoteScmSupported(Configuration.RemoteSCM.DIRBASED);
} else if (getopt.getOptarg().equalsIgnoreCase(UIONLY)) {
cfg.setRemoteScmSupported(Configuration.RemoteSCM.UIONLY);
} else {
System.err.println("ERROR: You should pass either \"on\" or \"off\" or \"uionly\" as argument to -r");
System.err.println(" Ex: \"-r on\" will allow retrieval for remote SCM systems");
System.err.println(" \"-r off\" will ignore SCM for remote systems");
System.err.println(" \"-r dirbased\" will allow retrieval during history index " + "only for repositories which allow getting history for directories");
System.err.println(" \"-r uionly\" will support remote SCM for UI only");
}
break;
case 'S':
searchRepositories = true;
break;
case 's':
{
File sourceRoot = new File(getopt.getOptarg());
if (!sourceRoot.isDirectory()) {
System.err.println("ERROR: Source root " + getopt.getOptarg() + " must be a directory");
System.exit(1);
}
cfg.setSourceRoot(sourceRoot.getCanonicalPath());
break;
}
case 'T':
try {
noThreads = Integer.parseInt(getopt.getOptarg());
} catch (NumberFormatException exp) {
System.err.println("ERROR: Failed to parse argument to \"-T\": " + exp.getMessage());
System.exit(1);
}
break;
case 't':
try {
int tmp = Integer.parseInt(getopt.getOptarg());
cfg.setTabSize(tmp);
} catch (NumberFormatException exp) {
System.err.println("ERROR: Failed to parse argument to \"-t\": " + exp.getMessage());
System.exit(1);
}
break;
case 'U':
configHost = getopt.getOptarg();
break;
case 'V':
System.out.println(Info.getFullVersion());
System.exit(0);
break;
case 'v':
cfg.setVerbose(true);
LoggerUtil.setBaseConsoleLogLevel(Level.INFO);
break;
case 'W':
configFilename = getopt.getOptarg();
break;
case 'w':
{
String webapp = getopt.getOptarg();
if (webapp.charAt(0) != '/' && !webapp.startsWith("http")) {
webapp = "/" + webapp;
}
if (webapp.endsWith("/")) {
cfg.setUrlPrefix(webapp + "s?");
} else {
cfg.setUrlPrefix(webapp + "/s?");
}
}
break;
case 'X':
cfg.setUserPageSuffix(getopt.getOptarg());
break;
case 'z':
try {
cfg.setScanningDepth(Integer.parseInt(getopt.getOptarg()));
} catch (NumberFormatException exp) {
System.err.println("ERROR: Failed to parse argument to \"-z\": " + exp.getMessage());
System.exit(1);
}
break;
case '?':
System.err.println(cmdOptions.getUsage());
System.exit(0);
break;
default:
System.err.println("Internal Error - Unimplemented cmdline option: " + (char) cmd);
System.exit(1);
}
}
if (configHost != null) {
String[] configHostArray = configHost.split(":");
if (configHostArray.length == 2) {
host = configHostArray[0];
try {
port = Integer.parseInt(configHostArray[1]);
} catch (NumberFormatException ex) {
System.err.println("Failed to parse: " + configHost);
System.exit(1);
}
} else {
System.err.println("Syntax error: ");
for (String s : configHostArray) {
System.err.println(s);
}
System.exit(1);
}
}
List<Class<? extends Repository>> repositoryClasses = RepositoryFactory.getRepositoryClasses();
for (Class<? extends Repository> clazz : repositoryClasses) {
try {
Field f = clazz.getDeclaredField("CMD_PROPERTY_KEY");
Object key = f.get(null);
if (key != null) {
cfg.setRepoCmd(clazz.getCanonicalName(), System.getProperty(key.toString()));
}
} catch (Exception e) {
// don't care
}
}
//logging starts here
if (cfg.isVerbose()) {
String fn = LoggerUtil.getFileHandlerPattern();
if (fn != null) {
System.out.println("Logging filehandler pattern: " + fn);
}
}
// automatically allow symlinks that are directly in source root
String file = cfg.getSourceRoot();
if (file != null) {
File sourceRootFile = new File(file);
File[] projectDirs = sourceRootFile.listFiles();
if (projectDirs != null) {
for (File projectDir : projectDirs) {
if (!projectDir.getCanonicalPath().equals(projectDir.getAbsolutePath())) {
allowedSymlinks.add(projectDir.getAbsolutePath());
}
}
}
}
allowedSymlinks.addAll(cfg.getAllowedSymlinks());
cfg.setAllowedSymlinks(allowedSymlinks);
// Assemble the unprocessed command line arguments (possibly
// a list of paths). This will be used to perform more fine
// grained checking in invalidateRepositories().
int optind = getopt.getOptind();
if (optind != -1) {
while (optind < argv.length) {
subFilesList.add(cfg.getSourceRoot() + argv[optind++]);
}
}
// Set updated configuration in RuntimeEnvironment.
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
env.setConfiguration(cfg, subFilesList);
/*
* Add paths to directories under source root. If projects
* are enabled the path should correspond to a project because
* project path is necessary to correctly set index directory
* (otherwise the index files will end up in index data root
* directory and not per project data root directory).
* For the check we need to have 'env' already set.
*/
for (String path : subFilesList) {
String srcPath = env.getSourceRootPath();
if (srcPath == null) {
System.err.println("Error getting source root from environment. Exiting.");
System.exit(1);
}
path = path.substring(srcPath.length());
if (env.hasProjects()) {
// The paths need to correspond to a project.
if (Project.getProject(path) != null) {
subFiles.add(path);
} else {
System.err.println("The path " + path + " does not correspond to a project");
}
} else {
subFiles.add(path);
}
}
if (!subFilesList.isEmpty() && subFiles.isEmpty()) {
System.err.println("None of the paths were added, exiting");
System.exit(1);
}
// Get history first.
getInstance().prepareIndexer(env, searchRepositories, addProjects, defaultProject, configFilename, refreshHistory, listFiles, createDict, subFiles, repositories, zapCache, listRepos);
if (listRepos || !zapCache.isEmpty()) {
return;
}
// And now index it all.
if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) {
IndexChangedListener progress = new DefaultIndexChangedListener();
getInstance().doIndexerExecution(update, noThreads, subFiles, progress);
}
// or send new configuration to the web application in the case of full reindex.
if (host != null) {
if (!subFiles.isEmpty()) {
getInstance().refreshSearcherManagers(env, subFiles, host, port);
} else {
getInstance().sendToConfigHost(env, host, port);
}
}
} catch (IndexerException ex) {
LOGGER.log(Level.SEVERE, "Exception running indexer", ex);
System.err.println(cmdOptions.getUsage());
System.exit(1);
} catch (Throwable e) {
System.err.println("Exception: " + e.getLocalizedMessage());
LOGGER.log(Level.SEVERE, "Unexpected Exception", e);
System.exit(1);
} finally {
stats.report(LOGGER);
}
}
}
use of org.opensolaris.opengrok.configuration.RuntimeEnvironment in project OpenGrok by OpenGrok.
the class Util method dumpConfiguration.
/**
* Dump the configuration as an HTML table.
*
* @param out destination for the HTML output
* @throws IOException if an error happens while writing to {@code out}
* @throws HistoryException if the history guru cannot be accesses
*/
@SuppressWarnings("boxing")
public static void dumpConfiguration(Appendable out) throws IOException, HistoryException {
out.append("<table border=\"1\" width=\"100%\">");
out.append("<tr><th>Variable</th><th>Value</th></tr>");
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
printTableRow(out, "Source root", env.getSourceRootPath());
printTableRow(out, "Data root", env.getDataRootPath());
printTableRow(out, "CTags", env.getCtags());
printTableRow(out, "Bug page", env.getBugPage());
printTableRow(out, "Bug pattern", env.getBugPattern());
printTableRow(out, "User page", env.getUserPage());
printTableRow(out, "User page suffix", env.getUserPageSuffix());
printTableRow(out, "Review page", env.getReviewPage());
printTableRow(out, "Review pattern", env.getReviewPattern());
printTableRow(out, "Using projects", env.hasProjects());
out.append("<tr><td>Ignored files</td><td>");
printUnorderedList(out, env.getIgnoredNames().getItems());
out.append("</td></tr>");
printTableRow(out, "lucene RAM_BUFFER_SIZE_MB", env.getRamBufferSize());
printTableRow(out, "Allow leading wildcard in search", env.isAllowLeadingWildcard());
printTableRow(out, "History cache", HistoryGuru.getInstance().getCacheInfo());
out.append("</table>");
}
Aggregations