Examples with StringField org.apache.lucene.document.StringField used on opensource projects

Example 1 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testHandleExceptinoOnSendSendFiles.

public void testHandleExceptinoOnSendSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
    Path tempDir = createTempDir();
    Store store = newStore(tempDir, false);
    AtomicBoolean failedEngine = new AtomicBoolean(false);
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {

        @Override
        protected void failEngine(IOException cause) {
            assertFalse(failedEngine.get());
            failedEngine.set(true);
        }
    };
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    final boolean throwCorruptedIndexException = randomBoolean();
    Store targetStore = newStore(createTempDir(), false);
    try {
        handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
            if (throwCorruptedIndexException) {
                throw new RuntimeException(new CorruptIndexException("foo", "bar"));
            } else {
                throw new RuntimeException("boom");
            }
        });
        fail("exception index");
    } catch (RuntimeException ex) {
        assertNull(ExceptionsHelper.unwrapCorruption(ex));
        if (throwCorruptedIndexException) {
            assertEquals(ex.getMessage(), "[File corruption occurred on recovery but checksums are ok]");
        } else {
            assertEquals(ex.getMessage(), "boom");
        }
    } catch (CorruptIndexException ex) {
        fail("not expected here");
    }
    assertFalse(failedEngine.get());
    IOUtils.close(store, targetStore);
}

Example 2 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class CategoryContextMappingTests method testParsingContextFromDocument.

public void testParsingContextFromDocument() throws Exception {
    CategoryContextMapping mapping = ContextBuilder.category("cat").field("category").build();
    ParseContext.Document document = new ParseContext.Document();
    document.add(new StringField("category", "category1", Field.Store.NO));
    Set<CharSequence> context = mapping.parseContext(document);
    assertThat(context.size(), equalTo(1));
    assertTrue(context.contains("category1"));
}

Example 3 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class CandidateQueryTests method testDuel.

public void testDuel() throws Exception {
    List<Function<String, Query>> queryFunctions = new ArrayList<>();
    queryFunctions.add((id) -> new PrefixQuery(new Term("field", id)));
    queryFunctions.add((id) -> new WildcardQuery(new Term("field", id + "*")));
    queryFunctions.add((id) -> new CustomQuery(new Term("field", id)));
    queryFunctions.add((id) -> new SpanTermQuery(new Term("field", id)));
    queryFunctions.add((id) -> new TermQuery(new Term("field", id)));
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        return builder.build();
    });
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST);
        if (randomBoolean()) {
            builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
        }
        if (randomBoolean()) {
            builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST);
        }
        return builder.build();
    });
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
        if (randomBoolean()) {
            builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
        }
        if (randomBoolean()) {
            builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
        }
        return builder.build();
    });
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
        if (randomBoolean()) {
            builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
        }
        return builder.build();
    });
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
        if (randomBoolean()) {
            builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
        }
        return builder.build();
    });
    queryFunctions.add((id) -> {
        BooleanQuery.Builder builder = new BooleanQuery.Builder();
        builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4));
        builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
        builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
        return builder.build();
    });
    queryFunctions.add((id) -> new MatchAllDocsQuery());
    queryFunctions.add((id) -> new MatchNoDocsQuery("no reason at all"));
    int numDocs = randomIntBetween(queryFunctions.size(), queryFunctions.size() * 3);
    List<ParseContext.Document> documents = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
        String id = Integer.toString(i);
        Query query = queryFunctions.get(i % queryFunctions.size()).apply(id);
        addQuery(query, documents);
    }
    indexWriter.addDocuments(documents);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    // Disable query cache, because ControlQuery cannot be cached...
    shardSearcher.setQueryCache(null);
    for (int i = 0; i < numDocs; i++) {
        String id = Integer.toString(i);
        Iterable<? extends IndexableField> doc = Collections.singleton(new StringField("field", id, Field.Store.NO));
        MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
        duelRun(queryStore, memoryIndex, shardSearcher);
    }
    Iterable<? extends IndexableField> doc = Collections.singleton(new StringField("field", "value", Field.Store.NO));
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
    // Empty percolator doc:
    memoryIndex = new MemoryIndex();
    duelRun(queryStore, memoryIndex, shardSearcher);
}

Example 4 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class PercolateQueryTests method testPercolateQuery.

public void testPercolateQuery() throws Exception {
    List<Iterable<? extends IndexableField>> docs = new ArrayList<>();
    List<Query> queries = new ArrayList<>();
    PercolateQuery.QueryStore queryStore = ctx -> queries::get;
    queries.add(new TermQuery(new Term("field", "fox")));
    docs.add(Collections.singleton(new StringField("select", "a", Field.Store.NO)));
    SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true);
    snp.addClause(new SpanTermQuery(new Term("field", "jumps")));
    snp.addClause(new SpanTermQuery(new Term("field", "lazy")));
    snp.addClause(new SpanTermQuery(new Term("field", "dog")));
    snp.setSlop(2);
    queries.add(snp.build());
    docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
    PhraseQuery.Builder pq1 = new PhraseQuery.Builder();
    pq1.add(new Term("field", "quick"));
    pq1.add(new Term("field", "brown"));
    pq1.add(new Term("field", "jumps"));
    pq1.setSlop(1);
    queries.add(pq1.build());
    docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
    BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
    bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST);
    bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST);
    bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST);
    queries.add(bq1.build());
    docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
    indexWriter.addDocuments(docs);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    MemoryIndex memoryIndex = new MemoryIndex();
    memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
    IndexSearcher percolateSearcher = memoryIndex.createSearcher();
    // no scoring, wrapping it in a constant score query:
    Query query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("a"), new TermQuery(new Term("select", "a")), percolateSearcher, new MatchNoDocsQuery("")));
    TopDocs topDocs = shardSearcher.search(query, 10);
    assertThat(topDocs.totalHits, equalTo(1));
    assertThat(topDocs.scoreDocs.length, equalTo(1));
    assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
    Explanation explanation = shardSearcher.explain(query, 0);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
    query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("b"), new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery("")));
    topDocs = shardSearcher.search(query, 10);
    assertThat(topDocs.totalHits, equalTo(3));
    assertThat(topDocs.scoreDocs.length, equalTo(3));
    assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
    explanation = shardSearcher.explain(query, 1);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
    assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
    explanation = shardSearcher.explain(query, 2);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score));
    assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
    explanation = shardSearcher.explain(query, 2);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score));
    query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("c"), new MatchAllDocsQuery(), percolateSearcher, new MatchAllDocsQuery()));
    topDocs = shardSearcher.search(query, 10);
    assertThat(topDocs.totalHits, equalTo(4));
    query = new PercolateQuery("type", queryStore, new BytesArray("{}"), new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery(""));
    topDocs = shardSearcher.search(query, 10);
    assertThat(topDocs.totalHits, equalTo(3));
    assertThat(topDocs.scoreDocs.length, equalTo(3));
    assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
    explanation = shardSearcher.explain(query, 3);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
    assertThat(explanation.getDetails(), arrayWithSize(1));
    assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
    explanation = shardSearcher.explain(query, 2);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score));
    assertThat(explanation.getDetails(), arrayWithSize(1));
    assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
    explanation = shardSearcher.explain(query, 1);
    assertThat(explanation.isMatch(), is(true));
    assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score));
    assertThat(explanation.getDetails(), arrayWithSize(1));
}

Example 5 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class AbstractStringFieldDataTestCase method testNestedSorting.

public void testNestedSorting(MultiValueMode sortMode) throws IOException {
    final String[] values = new String[randomIntBetween(2, 20)];
    for (int i = 0; i < values.length; ++i) {
        values[i] = TestUtil.randomSimpleString(random());
    }
    final int numParents = scaledRandomIntBetween(10, 3072);
    List<Document> docs = new ArrayList<>();
    FixedBitSet parents = new FixedBitSet(64);
    for (int i = 0; i < numParents; ++i) {
        docs.clear();
        final int numChildren = randomInt(4);
        for (int j = 0; j < numChildren; ++j) {
            final Document child = new Document();
            final int numValues = randomInt(3);
            for (int k = 0; k < numValues; ++k) {
                final String value = RandomPicks.randomFrom(random(), values);
                addField(child, "text", value);
            }
            docs.add(child);
        }
        final Document parent = new Document();
        parent.add(new StringField("type", "parent", Store.YES));
        final String value = RandomPicks.randomFrom(random(), values);
        if (value != null) {
            addField(parent, "text", value);
        }
        docs.add(parent);
        int bit = parents.prevSetBit(parents.length() - 1) + docs.size();
        parents = FixedBitSet.ensureCapacity(parents, bit);
        parents.set(bit);
        writer.addDocuments(docs);
        if (randomInt(10) == 0) {
            writer.commit();
        }
    }
    DirectoryReader directoryReader = DirectoryReader.open(writer);
    directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0));
    IndexSearcher searcher = new IndexSearcher(directoryReader);
    IndexFieldData<?> fieldData = getForField("text");
    final Object missingValue;
    switch(randomInt(4)) {
        case 0:
            missingValue = "_first";
            break;
        case 1:
            missingValue = "_last";
            break;
        case 2:
            missingValue = new BytesRef(RandomPicks.randomFrom(random(), values));
            break;
        default:
            missingValue = new BytesRef(TestUtil.randomSimpleString(random()));
            break;
    }
    Query parentFilter = new TermQuery(new Term("type", "parent"));
    Query childFilter = Queries.not(parentFilter);
    Nested nested = createNested(searcher, parentFilter, childFilter);
    BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested);
    ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None);
    Sort sort = new Sort(new SortField("text", nestedComparatorSource));
    TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
    assertTrue(topDocs.scoreDocs.length > 0);
    BytesRef previous = null;
    for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
        final int docID = topDocs.scoreDocs[i].doc;
        assertTrue("expected " + docID + " to be a parent", parents.get(docID));
        BytesRef cmpValue = null;
        for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
            String[] sVals = searcher.doc(child).getValues("text");
            final BytesRef[] vals;
            if (sVals.length == 0) {
                vals = new BytesRef[0];
            } else {
                vals = new BytesRef[sVals.length];
                for (int j = 0; j < vals.length; ++j) {
                    vals[j] = new BytesRef(sVals[j]);
                }
            }
            for (BytesRef value : vals) {
                if (cmpValue == null) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) {
                    cmpValue = value;
                }
            }
        }
        if (cmpValue == null) {
            if ("_first".equals(missingValue)) {
                cmpValue = new BytesRef();
            } else if ("_last".equals(missingValue) == false) {
                cmpValue = (BytesRef) missingValue;
            }
        }
        if (previous != null && cmpValue != null) {
            assertTrue(previous.utf8ToString() + "   /   " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
        }
        previous = cmpValue;
    }
    searcher.getIndexReader().close();
}