Examples with HistoryException org.opensolaris.opengrok.history.HistoryException used on opensource projects

Example 1 with HistoryException

use of org.opensolaris.opengrok.history.HistoryException in project OpenGrok by OpenGrok.

the class AnalyzerGuru method populateDocument.

/**
     * Populate a Lucene document with the required fields.
     *
     * @param doc The document to populate
     * @param file The file to index
     * @param path Where the file is located (from source root)
     * @param fa The analyzer to use on the file
     * @param xrefOut Where to write the xref (possibly {@code null})
     * @throws IOException If an exception occurs while collecting the data
     */
public void populateDocument(Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
    try {
        HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
        if (hr != null) {
            doc.add(new TextField(QueryBuilder.HIST, hr));
        // date = hr.getLastCommentDate() //RFE
        }
    } catch (HistoryException e) {
        LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    if (path != null) {
        doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
        Project project = Project.getProject(path);
        if (project != null) {
            doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
        }
    }
    if (fa != null) {
        Genre g = fa.getGenre();
        if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
            doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
        }
        fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
        String type = fa.getFileTypeName();
        doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
}

Example 2 with HistoryException

use of org.opensolaris.opengrok.history.HistoryException in project OpenGrok by OpenGrok.

the class Indexer method prepareIndexer.

/*
     * This is the first phase of the indexing where history cache is being
     * generated for repositories (at least for those which support getting
     * history per directory).
     *
     * PMD wants us to use length() > 0 && charAt(0) instead of startsWith()
     * for performance. We prefer clarity over performance here, so silence it.
     */
@SuppressWarnings("PMD.SimplifyStartsWith")
public void prepareIndexer(RuntimeEnvironment env, boolean searchRepositories, boolean addProjects, String defaultProject, String configFilename, boolean refreshHistory, boolean listFiles, boolean createDict, List<String> subFiles, List<String> repositories, List<String> zapCache, boolean listRepoPaths) throws IndexerException, IOException {
    if (env.getDataRootPath() == null) {
        throw new IndexerException("ERROR: Please specify a DATA ROOT path");
    }
    if (env.getSourceRootFile() == null) {
        throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");
    }
    if (zapCache.isEmpty() && !env.validateExuberantCtags()) {
        throw new IndexerException("Didn't find Exuberant Ctags");
    }
    if (zapCache == null) {
        throw new IndexerException("Internal error, zapCache shouldn't be null");
    }
    if (searchRepositories || listRepoPaths || !zapCache.isEmpty()) {
        LOGGER.log(Level.INFO, "Scanning for repositories...");
        long start = System.currentTimeMillis();
        if (refreshHistory == true) {
            HistoryGuru.getInstance().addRepositories(env.getSourceRootPath());
        }
        long time = (System.currentTimeMillis() - start) / 1000;
        LOGGER.log(Level.INFO, "Done scanning for repositories ({0}s)", time);
        if (listRepoPaths || !zapCache.isEmpty()) {
            List<RepositoryInfo> repos = env.getRepositories();
            String prefix = env.getSourceRootPath();
            if (listRepoPaths) {
                if (repos.isEmpty()) {
                    System.out.println("No repositories found.");
                    return;
                }
                System.out.println("Repositories in " + prefix + ":");
                for (RepositoryInfo info : env.getRepositories()) {
                    String dir = info.getDirectoryName();
                    System.out.println(dir.substring(prefix.length()));
                }
            }
            if (!zapCache.isEmpty()) {
                HashSet<String> toZap = new HashSet<>(zapCache.size() << 1);
                boolean all = false;
                for (String repo : zapCache) {
                    if ("*".equals(repo)) {
                        all = true;
                        break;
                    }
                    if (repo.startsWith(prefix)) {
                        repo = repo.substring(prefix.length());
                    }
                    toZap.add(repo);
                }
                if (all) {
                    toZap.clear();
                    for (RepositoryInfo info : env.getRepositories()) {
                        toZap.add(info.getDirectoryName().substring(prefix.length()));
                    }
                }
                try {
                    HistoryGuru.getInstance().removeCache(toZap);
                } catch (HistoryException e) {
                    LOGGER.log(Level.WARNING, "Clearing history cache failed: {0}", e.getLocalizedMessage());
                }
            }
            return;
        }
    }
    if (addProjects) {
        File[] files = env.getSourceRootFile().listFiles();
        List<Project> projects = env.getProjects();
        // Keep a copy of the old project list so that we can preserve
        // the customization of existing projects.
        Map<String, Project> oldProjects = new HashMap<>();
        for (Project p : projects) {
            oldProjects.put(p.getPath(), p);
        }
        projects.clear();
        // Add a project for each top-level directory in source root.
        for (File file : files) {
            String name = file.getName();
            String path = "/" + name;
            if (oldProjects.containsKey(path)) {
                // This is an existing object. Reuse the old project,
                // possibly with customizations, instead of creating a
                // new with default values.
                projects.add(oldProjects.get(path));
            } else if (!name.startsWith(".") && file.isDirectory()) {
                // Found a new directory with no matching project, so
                // create a new project with default properties.
                Project p = new Project();
                p.setName(name);
                p.setPath(path);
                p.setTabSize(env.getConfiguration().getTabSize());
                projects.add(p);
            }
        }
        // The projects should be sorted...
        Collections.sort(projects, new Comparator<Project>() {

            @Override
            public int compare(Project p1, Project p2) {
                String s1 = p1.getName();
                String s2 = p2.getName();
                int ret;
                if (s1 == null) {
                    ret = (s2 == null) ? 0 : 1;
                } else {
                    ret = s1.compareTo(s2);
                }
                return ret;
            }
        });
    }
    if (defaultProject != null) {
        for (Project p : env.getProjects()) {
            if (p.getPath().equals(defaultProject)) {
                env.setDefaultProject(p);
                break;
            }
        }
    }
    if (configFilename != null) {
        LOGGER.log(Level.INFO, "Writing configuration to {0}", configFilename);
        env.writeConfiguration(new File(configFilename));
        LOGGER.info("Done...");
    }
    if (refreshHistory) {
        if (repositories != null && !repositories.isEmpty()) {
            LOGGER.log(Level.INFO, "Generating history cache for repositories: " + repositories.stream().collect(Collectors.joining(",")));
            HistoryGuru.getInstance().createCache(repositories);
            LOGGER.info("Done...");
        } else {
            LOGGER.log(Level.INFO, "Generating history cache for all repositories ...");
            HistoryGuru.getInstance().createCache();
            LOGGER.info("Done...");
        }
    }
    if (listFiles) {
        IndexDatabase.listAllFiles(subFiles);
    }
    if (createDict) {
        IndexDatabase.listFrequentTokens(subFiles);
    }
}

Example 3 with HistoryException

use of org.opensolaris.opengrok.history.HistoryException in project OpenGrok by OpenGrok.

the class SearchEngine method results.

/**
     * get results , if no search was started before, no results are returned
     * this method will requery if end is more than first query from search,
     * hence performance hit applies, if you want results in later pages than
     * number of cachePages also end has to be bigger than start !
     *
     * @param start start of the hit list
     * @param end end of the hit list
     * @param ret list of results from start to end or null/empty if no search
     * was started
     */
public void results(int start, int end, List<Hit> ret) {
    //return if no start search() was done
    if (hits == null || (end < start)) {
        ret.clear();
        return;
    }
    ret.clear();
    //TODO check if below fits for if end=old hits.length, or it should include it
    if (end > hits.length & !allCollected) {
        //do the requery, we want more than 5 pages
        collector = TopScoreDocCollector.create(totalHits);
        try {
            searcher.search(query, collector);
        } catch (Exception e) {
            // this exception should never be hit, since search() will hit this before
            LOGGER.log(Level.WARNING, SEARCH_EXCEPTION_MSG, e);
        }
        hits = collector.topDocs().scoreDocs;
        Document d = null;
        for (int i = start; i < hits.length; i++) {
            int docId = hits[i].doc;
            try {
                d = searcher.doc(docId);
            } catch (Exception e) {
                LOGGER.log(Level.SEVERE, SEARCH_EXCEPTION_MSG, e);
            }
            docs.add(d);
        }
        allCollected = true;
    }
    // the only problem is that count of docs is usually smaller than number of results
    for (int ii = start; ii < end; ++ii) {
        boolean alt = (ii % 2 == 0);
        boolean hasContext = false;
        try {
            Document doc = docs.get(ii);
            String filename = doc.get(QueryBuilder.PATH);
            Genre genre = Genre.get(doc.get(QueryBuilder.T));
            Definitions tags = null;
            IndexableField tagsField = doc.getField(QueryBuilder.TAGS);
            if (tagsField != null) {
                tags = Definitions.deserialize(tagsField.binaryValue().bytes);
            }
            Scopes scopes = null;
            IndexableField scopesField = doc.getField(QueryBuilder.SCOPES);
            if (scopesField != null) {
                scopes = Scopes.deserialize(scopesField.binaryValue().bytes);
            }
            int nhits = docs.size();
            if (sourceContext != null) {
                try {
                    if (Genre.PLAIN == genre && (source != null)) {
                        hasContext = sourceContext.getContext(new InputStreamReader(new FileInputStream(source + filename)), null, null, null, filename, tags, nhits > 100, false, ret, scopes);
                    } else if (Genre.XREFABLE == genre && data != null && summarizer != null) {
                        int l;
                        try (Reader r = RuntimeEnvironment.getInstance().isCompressXref() ? new HTMLStripCharFilter(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(data + Prefix.XREF_P + filename + ".gz"))))) : new HTMLStripCharFilter(new BufferedReader(new FileReader(data + Prefix.XREF_P + filename)))) {
                            l = r.read(content);
                        }
                        //TODO FIX below fragmenter according to either summarizer or context (to get line numbers, might be hard, since xref writers will need to be fixed too, they generate just one line of html code now :( )
                        Summary sum = summarizer.getSummary(new String(content, 0, l));
                        Fragment[] fragments = sum.getFragments();
                        for (int jj = 0; jj < fragments.length; ++jj) {
                            String match = fragments[jj].toString();
                            if (match.length() > 0) {
                                if (!fragments[jj].isEllipsis()) {
                                    Hit hit = new Hit(filename, fragments[jj].toString(), "", true, alt);
                                    ret.add(hit);
                                }
                                hasContext = true;
                            }
                        }
                    } else {
                        LOGGER.log(Level.WARNING, "Unknown genre: {0} for {1}", new Object[] { genre, filename });
                        hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, false, ret, scopes);
                    }
                } catch (FileNotFoundException exp) {
                    LOGGER.log(Level.WARNING, "Couldn''t read summary from {0} ({1})", new Object[] { filename, exp.getMessage() });
                    hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, false, ret, scopes);
                }
            }
            if (historyContext != null) {
                hasContext |= historyContext.getContext(source + filename, filename, ret);
            }
            if (!hasContext) {
                ret.add(new Hit(filename, "...", "", false, alt));
            }
        } catch (IOException | ClassNotFoundException | HistoryException e) {
            LOGGER.log(Level.WARNING, SEARCH_EXCEPTION_MSG, e);
        }
    }
}