Search in sources :

Example 1 with Snippet

use of org.apache.lucene.search.uhighlight.Snippet in project OpenSearch by opensearch-project.

the class AnnotatedTextHighlighterTests method assertHighlightOneDoc.

private void assertHighlightOneDoc(String fieldName, String[] markedUpInputs, Query query, Locale locale, BreakIterator breakIterator, int noMatchSize, String[] expectedPassages) throws Exception {
    // Annotated fields wrap the usual analyzer with one that injects extra tokens
    Analyzer wrapperAnalyzer = new AnnotationAnalyzerWrapper(new StandardAnalyzer());
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(wrapperAnalyzer);
    iwc.setMergePolicy(newTieredMergePolicy(random()));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    FieldType ft = new FieldType(TextField.TYPE_STORED);
    if (randomBoolean()) {
        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    } else {
        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    }
    ft.freeze();
    Document doc = new Document();
    for (String input : markedUpInputs) {
        Field field = new Field(fieldName, "", ft);
        field.setStringValue(input);
        doc.add(field);
    }
    iw.addDocument(doc);
    DirectoryReader reader = iw.getReader();
    IndexSearcher searcher = newSearcher(reader);
    iw.close();
    AnnotatedText[] annotations = new AnnotatedText[markedUpInputs.length];
    for (int i = 0; i < markedUpInputs.length; i++) {
        annotations[i] = AnnotatedText.parse(markedUpInputs[i]);
    }
    AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer);
    hiliteAnalyzer.setAnnotations(annotations);
    AnnotatedPassageFormatter passageFormatter = new AnnotatedPassageFormatter(new DefaultEncoder());
    passageFormatter.setAnnotations(annotations);
    ArrayList<Object> plainTextForHighlighter = new ArrayList<>(annotations.length);
    for (int i = 0; i < annotations.length; i++) {
        plainTextForHighlighter.add(annotations[i].textMinusMarkup);
    }
    TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), 1, Sort.INDEXORDER);
    assertThat(topDocs.totalHits.value, equalTo(1L));
    String rawValue = Strings.collectionToDelimitedString(plainTextForHighlighter, String.valueOf(MULTIVAL_SEP_CHAR));
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, hiliteAnalyzer, null, passageFormatter, locale, breakIterator, "index", "text", query, noMatchSize, expectedPassages.length, name -> "text".equals(name), Integer.MAX_VALUE, Integer.MAX_VALUE);
    highlighter.setFieldMatcher((name) -> "text".equals(name));
    final Snippet[] snippets = highlighter.highlightField(getOnlyLeafReader(reader), topDocs.scoreDocs[0].doc, () -> rawValue);
    assertEquals(expectedPassages.length, snippets.length);
    for (int i = 0; i < snippets.length; i++) {
        assertEquals(expectedPassages[i], snippets[i].getText());
    }
    reader.close();
    dir.close();
}
Also used : IndexSearcher(org.apache.lucene.search.IndexSearcher) AnnotatedHighlighterAnalyzer(org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedHighlighterAnalyzer) ArrayList(java.util.ArrayList) AnnotatedHighlighterAnalyzer(org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedHighlighterAnalyzer) Analyzer(org.apache.lucene.analysis.Analyzer) StandardAnalyzer(org.apache.lucene.analysis.standard.StandardAnalyzer) Document(org.apache.lucene.document.Document) TopDocs(org.apache.lucene.search.TopDocs) Field(org.apache.lucene.document.Field) TextField(org.apache.lucene.document.TextField) DefaultEncoder(org.apache.lucene.search.highlight.DefaultEncoder) Directory(org.apache.lucene.store.Directory) DirectoryReader(org.apache.lucene.index.DirectoryReader) AnnotatedText(org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText) CustomUnifiedHighlighter(org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter) Snippet(org.apache.lucene.search.uhighlight.Snippet) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) FieldType(org.apache.lucene.document.FieldType) StandardAnalyzer(org.apache.lucene.analysis.standard.StandardAnalyzer) AnnotationAnalyzerWrapper(org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig)

Example 2 with Snippet

use of org.apache.lucene.search.uhighlight.Snippet in project OpenSearch by opensearch-project.

the class AnnotatedPassageFormatter method format.

@Override
public Snippet[] format(Passage[] passages, String content) {
    Snippet[] snippets = new Snippet[passages.length];
    int pos;
    int j = 0;
    for (Passage passage : passages) {
        AnnotationToken[] annotations = getIntersectingAnnotations(passage.getStartOffset(), passage.getEndOffset());
        MarkupPassage mergedMarkup = mergeAnnotations(annotations, passage);
        StringBuilder sb = new StringBuilder();
        pos = passage.getStartOffset();
        for (Markup markup : mergedMarkup.markups) {
            int start = markup.start;
            int end = markup.end;
            // its possible to have overlapping terms
            if (start > pos) {
                append(sb, content, pos, start);
            }
            if (end > pos) {
                sb.append("[");
                append(sb, content, Math.max(pos, start), end);
                sb.append("](");
                sb.append(markup.metadata);
                sb.append(")");
                pos = end;
            }
        }
        // its possible a "term" from the analyzer could span a sentence boundary.
        append(sb, content, pos, Math.max(pos, passage.getEndOffset()));
        // we remove the paragraph separator if present at the end of the snippet (we used it as separator between values)
        if (sb.charAt(sb.length() - 1) == HighlightUtils.PARAGRAPH_SEPARATOR) {
            sb.deleteCharAt(sb.length() - 1);
        } else if (sb.charAt(sb.length() - 1) == HighlightUtils.NULL_SEPARATOR) {
            sb.deleteCharAt(sb.length() - 1);
        }
        // and we trim the snippets too
        snippets[j++] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
    }
    return snippets;
}
Also used : Snippet(org.apache.lucene.search.uhighlight.Snippet) Passage(org.apache.lucene.search.uhighlight.Passage) AnnotationToken(org.opensearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken)

Example 3 with Snippet

use of org.apache.lucene.search.uhighlight.Snippet in project OpenSearch by opensearch-project.

the class UnifiedHighlighter method highlight.

@Override
public HighlightField highlight(FieldHighlightContext fieldContext) throws IOException {
    @SuppressWarnings("unchecked") Map<String, CustomUnifiedHighlighter> cache = (Map<String, CustomUnifiedHighlighter>) fieldContext.cache.computeIfAbsent(UnifiedHighlighter.class.getName(), k -> new HashMap<>());
    if (cache.containsKey(fieldContext.fieldName) == false) {
        cache.put(fieldContext.fieldName, buildHighlighter(fieldContext));
    }
    CustomUnifiedHighlighter highlighter = cache.get(fieldContext.fieldName);
    MappedFieldType fieldType = fieldContext.fieldType;
    SearchHighlightContext.Field field = fieldContext.field;
    FetchSubPhase.HitContext hitContext = fieldContext.hitContext;
    CheckedSupplier<String, IOException> loadFieldValues = () -> {
        List<Object> fieldValues = loadFieldValues(highlighter, fieldContext.context.getQueryShardContext(), fieldType, hitContext, fieldContext.forceSource);
        if (fieldValues.size() == 0) {
            return null;
        }
        return mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
    };
    Snippet[] fieldSnippets = highlighter.highlightField(hitContext.reader(), hitContext.docId(), loadFieldValues);
    if (fieldSnippets == null || fieldSnippets.length == 0) {
        return null;
    }
    List<Snippet> snippets = new ArrayList<>(fieldSnippets.length);
    for (Snippet fieldSnippet : fieldSnippets) {
        if (Strings.hasText(fieldSnippet.getText())) {
            snippets.add(fieldSnippet);
        }
    }
    if (snippets.isEmpty()) {
        return null;
    }
    if (field.fieldOptions().scoreOrdered()) {
        // let's sort the snippets by score if needed
        CollectionUtil.introSort(snippets, (o1, o2) -> Double.compare(o2.getScore(), o1.getScore()));
    }
    String[] fragments = new String[snippets.size()];
    for (int i = 0; i < fragments.length; i++) {
        fragments[i] = snippets.get(i).getText();
    }
    return new HighlightField(fieldContext.fieldName, Text.convertFromStringArray(fragments));
}
Also used : IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) HashMap(java.util.HashMap) CustomPassageFormatter(org.apache.lucene.search.uhighlight.CustomPassageFormatter) Strings(org.opensearch.common.Strings) ArrayList(java.util.ArrayList) CheckedSupplier(org.opensearch.common.CheckedSupplier) Locale(java.util.Locale) Snippet(org.apache.lucene.search.uhighlight.Snippet) Map(java.util.Map) CustomUnifiedHighlighter(org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter) BoundedBreakIteratorScanner(org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner) CustomSeparatorBreakIterator(org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator) KeywordFieldMapper(org.opensearch.index.mapper.KeywordFieldMapper) HitContext(org.opensearch.search.fetch.FetchSubPhase.HitContext) MULTIVAL_SEP_CHAR(org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR) FetchSubPhase(org.opensearch.search.fetch.FetchSubPhase) OffsetSource(org.apache.lucene.search.uhighlight.UnifiedHighlighter.OffsetSource) BytesRef(org.apache.lucene.util.BytesRef) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) Predicate(java.util.function.Predicate) Analyzer(org.apache.lucene.analysis.Analyzer) TextSearchInfo(org.opensearch.index.mapper.TextSearchInfo) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) DocumentMapper(org.opensearch.index.mapper.DocumentMapper) CollectionUtil(org.apache.lucene.util.CollectionUtil) List(java.util.List) Encoder(org.apache.lucene.search.highlight.Encoder) BreakIterator(java.text.BreakIterator) QueryShardContext(org.opensearch.index.query.QueryShardContext) PassageFormatter(org.apache.lucene.search.uhighlight.PassageFormatter) Text(org.opensearch.common.text.Text) IndexSearcher(org.apache.lucene.search.IndexSearcher) HashMap(java.util.HashMap) CustomUnifiedHighlighter(org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter) ArrayList(java.util.ArrayList) IOException(java.io.IOException) Snippet(org.apache.lucene.search.uhighlight.Snippet) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) FetchSubPhase(org.opensearch.search.fetch.FetchSubPhase) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) HitContext(org.opensearch.search.fetch.FetchSubPhase.HitContext)

Aggregations

Snippet (org.apache.lucene.search.uhighlight.Snippet)3 ArrayList (java.util.ArrayList)2 Analyzer (org.apache.lucene.analysis.Analyzer)2 IndexSearcher (org.apache.lucene.search.IndexSearcher)2 CustomUnifiedHighlighter (org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter)2 IOException (java.io.IOException)1 BreakIterator (java.text.BreakIterator)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Locale (java.util.Locale)1 Map (java.util.Map)1 Predicate (java.util.function.Predicate)1 Collectors (java.util.stream.Collectors)1 StandardAnalyzer (org.apache.lucene.analysis.standard.StandardAnalyzer)1 Document (org.apache.lucene.document.Document)1 Field (org.apache.lucene.document.Field)1 FieldType (org.apache.lucene.document.FieldType)1 TextField (org.apache.lucene.document.TextField)1 DirectoryReader (org.apache.lucene.index.DirectoryReader)1 IndexWriterConfig (org.apache.lucene.index.IndexWriterConfig)1