Examples with Genre org.opensolaris.opengrok.analysis.FileAnalyzer.Genre used on opensource projects

Example 1 with Genre

use of org.opensolaris.opengrok.analysis.FileAnalyzer.Genre in project OpenGrok by OpenGrok.

the class Results method prettyPrint.

/**
     * Prints out results in html form. The following search helper fields are
     * required to be properly initialized: <ul>
     * <li>{@link SearchHelper#dataRoot}</li>
     * <li>{@link SearchHelper#contextPath}</li>
     * <li>{@link SearchHelper#searcher}</li> <li>{@link SearchHelper#hits}</li>
     * <li>{@link SearchHelper#historyContext} (ignored if {@code null})</li>
     * <li>{@link SearchHelper#sourceContext} (ignored if {@code null})</li>
     * <li>{@link SearchHelper#summarizer} (if sourceContext is not
     * {@code null})</li> <li>{@link SearchHelper#compressed} (if sourceContext
     * is not {@code null})</li> <li>{@link SearchHelper#sourceRoot} (if
     * sourceContext or historyContext is not {@code null})</li> </ul>
     *
     * @param out write destination
     * @param sh search helper which has all required fields set
     * @param start index of the first hit to print
     * @param end index of the last hit to print
     * @throws HistoryException
     * @throws IOException
     * @throws ClassNotFoundException
     */
public static void prettyPrint(Writer out, SearchHelper sh, int start, int end) throws HistoryException, IOException, ClassNotFoundException {
    Project p;
    String ctxE = Util.URIEncodePath(sh.contextPath);
    String xrefPrefix = sh.contextPath + Prefix.XREF_P;
    String morePrefix = sh.contextPath + Prefix.MORE_P;
    String xrefPrefixE = ctxE + Prefix.XREF_P;
    File xrefDataDir = new File(sh.dataRoot, Prefix.XREF_P.toString());
    for (Map.Entry<String, ArrayList<Document>> entry : createMap(sh.searcher, sh.hits, start, end).entrySet()) {
        String parent = entry.getKey();
        out.write("<tr class=\"dir\"><td colspan=\"3\"><a href=\"");
        out.write(xrefPrefixE);
        out.write(Util.URIEncodePath(parent));
        out.write("/\">");
        // htmlize ???
        out.write(parent);
        out.write("/</a>");
        if (sh.desc != null) {
            out.write(" - <i>");
            // htmlize ???
            out.write(sh.desc.get(parent));
            out.write("</i>");
        }
        JSONArray messages;
        if ((p = Project.getProject(parent)) != null && (messages = Util.messagesToJson(p, RuntimeEnvironment.MESSAGES_MAIN_PAGE_TAG)).size() > 0) {
            out.write(" <a ");
            out.write("href=\"" + xrefPrefix + "/" + p.getName() + "\">");
            out.write("<span class=\"important-note important-note-rounded\" data-messages='" + messages + "'>!</span>");
            out.write("</a>");
        }
        out.write("</td></tr>");
        for (Document doc : entry.getValue()) {
            String rpath = doc.get(QueryBuilder.PATH);
            String rpathE = Util.URIEncodePath(rpath);
            DateFormat df;
            out.write("<tr>");
            Util.writeHAD(out, sh.contextPath, rpathE, false);
            out.write("<td class=\"f\"><a href=\"");
            out.write(xrefPrefixE);
            out.write(rpathE);
            out.write("\"");
            if (RuntimeEnvironment.getInstance().isLastEditedDisplayMode()) {
                try {
                    // insert last edited date if possible
                    df = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
                    String dd = df.format(DateTools.stringToDate(doc.get("date")));
                    out.write(" class=\"result-annotate\" title=\"");
                    out.write("Last modified: ");
                    out.write(dd);
                    out.write("\"");
                } catch (ParseException ex) {
                    LOGGER.log(Level.WARNING, "An error parsing date information", ex);
                }
            }
            out.write(">");
            // htmlize ???
            out.write(rpath.substring(rpath.lastIndexOf('/') + 1));
            out.write("</a>");
            out.write("</td><td><tt class=\"con\">");
            if (sh.sourceContext != null) {
                Genre genre = Genre.get(doc.get("t"));
                Definitions tags = null;
                IndexableField tagsField = doc.getField(QueryBuilder.TAGS);
                if (tagsField != null) {
                    tags = Definitions.deserialize(tagsField.binaryValue().bytes);
                }
                Scopes scopes;
                IndexableField scopesField = doc.getField(QueryBuilder.SCOPES);
                if (scopesField != null) {
                    scopes = Scopes.deserialize(scopesField.binaryValue().bytes);
                } else {
                    scopes = new Scopes();
                }
                if (Genre.XREFABLE == genre && sh.summarizer != null) {
                    String xtags = getTags(xrefDataDir, rpath, sh.compressed);
                    // FIXME use Highlighter from lucene contrib here,
                    // instead of summarizer, we'd also get rid of
                    // apache lucene in whole source ...
                    out.write(sh.summarizer.getSummary(xtags).toString());
                } else if (Genre.HTML == genre && sh.summarizer != null) {
                    String htags = getTags(sh.sourceRoot, rpath, false);
                    out.write(sh.summarizer.getSummary(htags).toString());
                } else {
                    FileReader r = genre == Genre.PLAIN ? new FileReader(new File(sh.sourceRoot, rpath)) : null;
                    sh.sourceContext.getContext(r, out, xrefPrefix, morePrefix, rpath, tags, true, sh.builder.isDefSearch(), null, scopes);
                }
            }
            if (sh.historyContext != null) {
                sh.historyContext.getContext(new File(sh.sourceRoot, rpath), rpath, out, sh.contextPath);
            }
            out.write("</tt></td></tr>\n");
        }
    }
}

Example 2 with Genre

use of org.opensolaris.opengrok.analysis.FileAnalyzer.Genre in project OpenGrok by OpenGrok.

the class Results method prettyPrint.

/**
 * Prints out results in html form. The following search helper fields are
 * required to be properly initialized: <ul>
 * <li>{@link SearchHelper#dataRoot}</li>
 * <li>{@link SearchHelper#contextPath}</li>
 * <li>{@link SearchHelper#searcher}</li> <li>{@link SearchHelper#hits}</li>
 * <li>{@link SearchHelper#historyContext} (ignored if {@code null})</li>
 * <li>{@link SearchHelper#sourceContext} (ignored if {@code null})</li>
 * <li>{@link SearchHelper#summarizer} (if sourceContext is not
 * {@code null})</li> <li>{@link SearchHelper#compressed} (if sourceContext
 * is not {@code null})</li> <li>{@link SearchHelper#sourceRoot} (if
 * sourceContext or historyContext is not {@code null})</li> </ul>
 *
 * @param out write destination
 * @param sh search helper which has all required fields set
 * @param start index of the first hit to print
 * @param end index of the last hit to print
 * @throws HistoryException
 * @throws IOException
 * @throws ClassNotFoundException
 */
public static void prettyPrint(Writer out, SearchHelper sh, int start, long end) throws HistoryException, IOException, ClassNotFoundException {
    Project p;
    String ctxE = Util.URIEncodePath(sh.contextPath);
    String xrefPrefix = sh.contextPath + Prefix.XREF_P;
    String morePrefix = sh.contextPath + Prefix.MORE_P;
    String xrefPrefixE = ctxE + Prefix.XREF_P;
    File xrefDataDir = new File(sh.dataRoot, Prefix.XREF_P.toString());
    for (Map.Entry<String, ArrayList<Document>> entry : createMap(sh.searcher, sh.hits, start, end).entrySet()) {
        String parent = entry.getKey();
        out.write("<tr class=\"dir\"><td colspan=\"3\"><a href=\"");
        out.write(xrefPrefixE);
        out.write(Util.URIEncodePath(parent));
        out.write("/\">");
        out.write(htmlize(parent));
        out.write("/</a>");
        if (sh.desc != null) {
            out.write(" - <i>");
            out.write(htmlize(sh.desc.get(parent)));
            out.write("</i>");
        }
        JSONArray messages;
        if ((p = Project.getProject(parent)) != null && (messages = Util.messagesToJson(p, RuntimeEnvironment.MESSAGES_MAIN_PAGE_TAG)).size() > 0) {
            out.write(" <a ");
            out.write("href=\"" + xrefPrefix + "/" + p.getName() + "\">");
            out.write("<span class=\"important-note important-note-rounded\" data-messages='" + messages + "'>!</span>");
            out.write("</a>");
        }
        out.write("</td></tr>");
        for (Document doc : entry.getValue()) {
            String rpath = doc.get(QueryBuilder.PATH);
            String rpathE = Util.URIEncodePath(rpath);
            DateFormat df;
            out.write("<tr>");
            Util.writeHAD(out, sh.contextPath, rpathE, false);
            out.write("<td class=\"f\"><a href=\"");
            out.write(xrefPrefixE);
            out.write(rpathE);
            out.write("\"");
            if (RuntimeEnvironment.getInstance().isLastEditedDisplayMode()) {
                try {
                    // insert last edited date if possible
                    df = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
                    String dd = df.format(DateTools.stringToDate(doc.get("date")));
                    out.write(" class=\"result-annotate\" title=\"");
                    out.write("Last modified: ");
                    out.write(dd);
                    out.write("\"");
                } catch (ParseException ex) {
                    LOGGER.log(Level.WARNING, "An error parsing date information", ex);
                }
            }
            out.write(">");
            out.write(htmlize(rpath.substring(rpath.lastIndexOf('/') + 1)));
            out.write("</a>");
            out.write("</td><td><tt class=\"con\">");
            if (sh.sourceContext != null) {
                Genre genre = Genre.get(doc.get("t"));
                Definitions tags = null;
                IndexableField tagsField = doc.getField(QueryBuilder.TAGS);
                if (tagsField != null) {
                    tags = Definitions.deserialize(tagsField.binaryValue().bytes);
                }
                Scopes scopes;
                IndexableField scopesField = doc.getField(QueryBuilder.SCOPES);
                if (scopesField != null) {
                    scopes = Scopes.deserialize(scopesField.binaryValue().bytes);
                } else {
                    scopes = new Scopes();
                }
                if (Genre.XREFABLE == genre && sh.summarizer != null) {
                    String xtags = getTags(xrefDataDir, rpath, sh.compressed);
                    // FIXME use Highlighter from lucene contrib here,
                    // instead of summarizer, we'd also get rid of
                    // apache lucene in whole source ...
                    out.write(sh.summarizer.getSummary(xtags).toString());
                } else if (Genre.HTML == genre && sh.summarizer != null) {
                    String htags = getTags(sh.sourceRoot, rpath, false);
                    out.write(sh.summarizer.getSummary(htags).toString());
                } else {
                    // SRCROOT is read with UTF-8 as a default.
                    Reader r = genre == Genre.PLAIN ? IOUtils.createBOMStrippedReader(new FileInputStream(new File(sh.sourceRoot, rpath)), StandardCharsets.UTF_8.name()) : null;
                    sh.sourceContext.getContext(r, out, xrefPrefix, morePrefix, rpath, tags, true, sh.builder.isDefSearch(), null, scopes);
                }
            }
            if (sh.historyContext != null) {
                sh.historyContext.getContext(new File(sh.sourceRoot, rpath), rpath, out, sh.contextPath);
            }
            out.write("</tt></td></tr>\n");
        }
    }
}

Example 3 with Genre

use of org.opensolaris.opengrok.analysis.FileAnalyzer.Genre in project OpenGrok by OpenGrok.

the class AnalyzerGuru method populateDocument.

/**
 * Populate a Lucene document with the required fields.
 *
 * @param doc The document to populate
 * @param file The file to index
 * @param path Where the file is located (from source root)
 * @param fa The analyzer to use on the file
 * @param xrefOut Where to write the xref (possibly {@code null})
 * @throws IOException If an exception occurs while collecting the data
 * @throws InterruptedException if a timeout occurs
 * @throws ForbiddenSymlinkException if symbolic-link checking encounters
 * an ineligible link
 */
public void populateDocument(Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException, InterruptedException, ForbiddenSymlinkException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
    if (RuntimeEnvironment.getInstance().isHistoryEnabled()) {
        try {
            HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
            if (hr != null) {
                doc.add(new TextField(QueryBuilder.HIST, hr));
            // date = hr.getLastCommentDate() //RFE
            }
        } catch (HistoryException e) {
            LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
        }
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    // `path' is not null, as it was passed to Util.path2uid() above.
    doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
    Project project = Project.getProject(path);
    if (project != null) {
        doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
    }
    /*
         * Use the parent of the path -- not the absolute file as is done for
         * FULLPATH -- so that DIRPATH is the same convention as for PATH
         * above. A StringField, however, is used instead of a TextField.
         */
    File fpath = new File(path);
    String fileParent = fpath.getParent();
    if (fileParent != null && fileParent.length() > 0) {
        String normalizedPath = QueryBuilder.normalizeDirPath(fileParent);
        StringField npstring = new StringField(QueryBuilder.DIRPATH, normalizedPath, Store.NO);
        doc.add(npstring);
    }
    if (fa != null) {
        Genre g = fa.getGenre();
        if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
            doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
        }
        fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
        String type = fa.getFileTypeName();
        doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
}

Example 4 with Genre

use of org.opensolaris.opengrok.analysis.FileAnalyzer.Genre in project OpenGrok by OpenGrok.

the class SearchEngine method results.

/**
 * get results , if no search was started before, no results are returned
 * this method will requery if end is more than first query from search,
 * hence performance hit applies, if you want results in later pages than
 * number of cachePages also end has to be bigger than start !
 *
 * @param start start of the hit list
 * @param end end of the hit list
 * @param ret list of results from start to end or null/empty if no search
 * was started
 */
public void results(int start, int end, List<Hit> ret) {
    // return if no start search() was done
    if (hits == null || (end < start)) {
        ret.clear();
        return;
    }
    ret.clear();
    // TODO check if below fits for if end=old hits.length, or it should include it
    if (end > hits.length & !allCollected) {
        // do the requery, we want more than 5 pages
        collector = TopScoreDocCollector.create(totalHits);
        try {
            searcher.search(query, collector);
        } catch (Exception e) {
            // this exception should never be hit, since search() will hit this before
            LOGGER.log(Level.WARNING, SEARCH_EXCEPTION_MSG, e);
        }
        hits = collector.topDocs().scoreDocs;
        Document d = null;
        for (int i = start; i < hits.length; i++) {
            int docId = hits[i].doc;
            try {
                d = searcher.doc(docId);
            } catch (Exception e) {
                LOGGER.log(Level.SEVERE, SEARCH_EXCEPTION_MSG, e);
            }
            docs.add(d);
        }
        allCollected = true;
    }
    // the only problem is that count of docs is usually smaller than number of results
    for (int ii = start; ii < end; ++ii) {
        boolean alt = (ii % 2 == 0);
        boolean hasContext = false;
        try {
            Document doc = docs.get(ii);
            String filename = doc.get(QueryBuilder.PATH);
            Genre genre = Genre.get(doc.get(QueryBuilder.T));
            Definitions tags = null;
            IndexableField tagsField = doc.getField(QueryBuilder.TAGS);
            if (tagsField != null) {
                tags = Definitions.deserialize(tagsField.binaryValue().bytes);
            }
            Scopes scopes = null;
            IndexableField scopesField = doc.getField(QueryBuilder.SCOPES);
            if (scopesField != null) {
                scopes = Scopes.deserialize(scopesField.binaryValue().bytes);
            }
            int nhits = docs.size();
            if (sourceContext != null) {
                try {
                    if (Genre.PLAIN == genre && (source != null)) {
                        // SRCROOT is read with UTF-8 as a default.
                        hasContext = sourceContext.getContext(new InputStreamReader(new FileInputStream(source + filename), StandardCharsets.UTF_8), null, null, null, filename, tags, nhits > 100, false, ret, scopes);
                    } else if (Genre.XREFABLE == genre && data != null && summarizer != null) {
                        int l;
                        /**
                         * For backward compatibility, read the
                         * OpenGrok-produced document using the system
                         * default charset.
                         */
                        try (Reader r = RuntimeEnvironment.getInstance().isCompressXref() ? new HTMLStripCharFilter(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(data + Prefix.XREF_P + filename + ".gz"))))) : new HTMLStripCharFilter(new BufferedReader(new FileReader(data + Prefix.XREF_P + filename)))) {
                            l = r.read(content);
                        }
                        // TODO FIX below fragmenter according to either summarizer or context (to get line numbers, might be hard, since xref writers will need to be fixed too, they generate just one line of html code now :( )
                        Summary sum = summarizer.getSummary(new String(content, 0, l));
                        Fragment[] fragments = sum.getFragments();
                        for (int jj = 0; jj < fragments.length; ++jj) {
                            String match = fragments[jj].toString();
                            if (match.length() > 0) {
                                if (!fragments[jj].isEllipsis()) {
                                    Hit hit = new Hit(filename, fragments[jj].toString(), "", true, alt);
                                    ret.add(hit);
                                }
                                hasContext = true;
                            }
                        }
                    } else {
                        LOGGER.log(Level.WARNING, "Unknown genre: {0} for {1}", new Object[] { genre, filename });
                        hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, false, ret, scopes);
                    }
                } catch (FileNotFoundException exp) {
                    LOGGER.log(Level.WARNING, "Couldn''t read summary from {0} ({1})", new Object[] { filename, exp.getMessage() });
                    hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, false, ret, scopes);
                }
            }
            if (historyContext != null) {
                hasContext |= historyContext.getContext(source + filename, filename, ret);
            }
            if (!hasContext) {
                ret.add(new Hit(filename, "...", "", false, alt));
            }
        } catch (IOException | ClassNotFoundException | HistoryException e) {
            LOGGER.log(Level.WARNING, SEARCH_EXCEPTION_MSG, e);
        }
    }
}

Example 5 with Genre

use of org.opensolaris.opengrok.analysis.FileAnalyzer.Genre in project OpenGrok by OpenGrok.

the class IndexDatabase method getXrefWriter.

/**
 * Get a writer to which the xref can be written, or null if no xref
 * should be produced for files of this type.
 */
private Writer getXrefWriter(FileAnalyzer fa, String path) throws IOException {
    Genre g = fa.getFactory().getGenre();
    if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
        boolean compressed = env.isCompressXref();
        File xrefFile = new File(xrefDir, path + (compressed ? ".gz" : ""));
        File parentFile = xrefFile.getParentFile();
        // only increase the file IO...
        if (!parentFile.mkdirs()) {
            assert parentFile.exists();
        }
        // Write to a pending file for later renaming.
        String xrefAbs = xrefFile.getAbsolutePath();
        File transientXref = new File(xrefAbs + PendingFileCompleter.PENDING_EXTENSION);
        PendingFileRenaming ren = new PendingFileRenaming(xrefAbs, transientXref.getAbsolutePath());
        completer.add(ren);
        return new BufferedWriter(new OutputStreamWriter(compressed ? new GZIPOutputStream(new FileOutputStream(transientXref)) : new FileOutputStream(transientXref)));
    }
    // no Xref for this analyzer
    return null;
}