Examples with NumLinesLOC org.opengrok.indexer.analysis.NumLinesLOC used on opensource projects

Example 1 with NumLinesLOC

use of org.opengrok.indexer.analysis.NumLinesLOC in project OpenGrok by OpenGrok.

the class MandocAnalyzer method analyze.

@Override
public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
    // this is to explicitly use appropriate analyzers tokenstream to
    // workaround #1376 symbols search works like full text search
    JFlexTokenizer symbolTokenizer = symbolTokenizerFactory.get();
    symbolTokenizer.setReader(getReader(src.getStream()));
    OGKTextField full = new OGKTextField(QueryBuilder.FULL, symbolTokenizer);
    doc.add(full);
    if (xrefOut != null) {
        try (Reader in = getReader(src.getStream())) {
            WriteXrefArgs args = new WriteXrefArgs(in, xrefOut);
            args.setProject(project);
            Xrefer xref = writeXref(args);
            String path = doc.get(QueryBuilder.PATH);
            addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
        }
    }
}

Example 2 with NumLinesLOC

use of org.opengrok.indexer.analysis.NumLinesLOC in project OpenGrok by OpenGrok.

the class TroffAnalyzer method analyze.

@Override
public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
    // this is to explicitly use appropriate analyzers tokenstream to workaround #1376 symbols search works like full text search
    JFlexTokenizer symbolTokenizer = symbolTokenizerFactory.get();
    symbolTokenizer.setReader(getReader(src.getStream()));
    OGKTextField full = new OGKTextField(QueryBuilder.FULL, symbolTokenizer);
    doc.add(full);
    if (xrefOut != null) {
        try (Reader in = getReader(src.getStream())) {
            WriteXrefArgs args = new WriteXrefArgs(in, xrefOut);
            args.setProject(project);
            Xrefer xref = writeXref(args);
            String path = doc.get(QueryBuilder.PATH);
            addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
        }
    }
}

Example 3 with NumLinesLOC

use of org.opengrok.indexer.analysis.NumLinesLOC in project OpenGrok by OpenGrok.

the class IndexDatabase method removeFile.

/**
 * Remove a stale file (uidIter.term().text()) from the index database and
 * history cache, and queue the removal of xref.
 *
 * @param removeHistory if false, do not remove history cache for this file
 * @throws java.io.IOException if an error occurs
 */
private void removeFile(boolean removeHistory) throws IOException {
    String path = Util.uid2url(uidIter.term().utf8ToString());
    for (IndexChangedListener listener : listeners) {
        listener.fileRemove(path);
    }
    // Determine if a reversal of counts is necessary, and execute if so.
    if (isCountingDeltas) {
        postsIter = uidIter.postings(postsIter);
        while (postsIter.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
            // Read a limited-fields version of the document.
            Document doc = reader.document(postsIter.docID(), REVERT_COUNTS_FIELDS);
            if (doc != null) {
                NullableNumLinesLOC nullableCounts = NumLinesLOCUtil.read(doc);
                if (nullableCounts.getNumLines() != null && nullableCounts.getLOC() != null) {
                    NumLinesLOC counts = new NumLinesLOC(path, -nullableCounts.getNumLines(), -nullableCounts.getLOC());
                    countsAggregator.register(counts);
                }
                break;
            }
        }
    }
    writer.deleteDocuments(new Term(QueryBuilder.U, uidIter.term()));
    removeXrefFile(path);
    if (removeHistory) {
        removeHistoryFile(path);
    }
    setDirty();
    for (IndexChangedListener listener : listeners) {
        listener.fileRemoved(path);
    }
}

Example 4 with NumLinesLOC

use of org.opengrok.indexer.analysis.NumLinesLOC in project OpenGrok by OpenGrok.

the class PlainAnalyzer method analyze.

@Override
public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException, InterruptedException {
    Definitions defs = null;
    NullWriter nullWriter = null;
    doc.add(new OGKTextField(QueryBuilder.FULL, getReader(src.getStream())));
    String fullPath = doc.get(QueryBuilder.FULLPATH);
    if (fullPath != null && ctags != null) {
        defs = ctags.doCtags(fullPath);
        if (defs != null && defs.numberOfSymbols() > 0) {
            tryAddingDefs(doc, defs, src);
            byte[] tags = defs.serialize();
            doc.add(new StoredField(QueryBuilder.TAGS, tags));
        }
    }
    /*
         * This is to explicitly use appropriate analyzer's token stream to
         * work around #1376: symbols search works like full text search.
         */
    JFlexTokenizer symbolTokenizer = symbolTokenizerFactory.get();
    OGKTextField ref = new OGKTextField(QueryBuilder.REFS, symbolTokenizer);
    symbolTokenizer.setReader(getReader(src.getStream()));
    doc.add(ref);
    if (scopesEnabled && xrefOut == null) {
        /*
             * Scopes are generated during xref generation. If xrefs are
             * turned off we still need to run writeXref() to produce scopes,
             * we use a dummy writer that will throw away any xref output.
             */
        nullWriter = new NullWriter();
        xrefOut = nullWriter;
    }
    if (xrefOut != null) {
        try (Reader in = getReader(src.getStream())) {
            RuntimeEnvironment env = RuntimeEnvironment.getInstance();
            WriteXrefArgs args = new WriteXrefArgs(in, xrefOut);
            args.setDefs(defs);
            args.setProject(project);
            CompletableFuture<XrefWork> future = CompletableFuture.supplyAsync(() -> {
                try {
                    return new XrefWork(writeXref(args));
                } catch (IOException e) {
                    return new XrefWork(e);
                }
            }, env.getIndexerParallelizer().getXrefWatcherExecutor()).orTimeout(env.getXrefTimeout(), TimeUnit.SECONDS);
            // Will throw ExecutionException wrapping TimeoutException on timeout.
            XrefWork xrefWork = future.get();
            Xrefer xref = xrefWork.xrefer;
            if (xref != null) {
                Scopes scopes = xref.getScopes();
                if (scopes.size() > 0) {
                    byte[] scopesSerialized = scopes.serialize();
                    doc.add(new StoredField(QueryBuilder.SCOPES, scopesSerialized));
                }
                String path = doc.get(QueryBuilder.PATH);
                addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
            } else {
                // Re-throw the exception from writeXref().
                throw new IOException(xrefWork.exception);
            }
        } catch (ExecutionException e) {
            throw new InterruptedException("failed to generate xref :" + e);
        } finally {
            if (nullWriter != null) {
                nullWriter.close();
            }
        }
    }
}

Example 5 with NumLinesLOC

use of org.opengrok.indexer.analysis.NumLinesLOC in project OpenGrok by OpenGrok.

the class XMLAnalyzer method analyze.

@Override
public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOException {
    doc.add(new OGKTextField(QueryBuilder.FULL, getReader(src.getStream())));
    if (xrefOut != null) {
        try (Reader in = getReader(src.getStream())) {
            WriteXrefArgs args = new WriteXrefArgs(in, xrefOut);
            args.setProject(project);
            Xrefer xref = writeXref(args);
            String path = doc.get(QueryBuilder.PATH);
            addNumLinesLOC(doc, new NumLinesLOC(path, xref.getLineNumber(), xref.getLOC()));
        }
    }
}