Examples with Directory org.apache.lucene.store.Directory used on opensource projects

Example 56 with Directory

use of org.apache.lucene.store.Directory in project elasticsearch by elastic.

the class XMoreLikeThisTests method testTopN.

public void testTopN() throws Exception {
    int numDocs = 100;
    int topN = 25;
    // add series of docs with terms of decreasing df
    Directory dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    for (int i = 0; i < numDocs; i++) {
        addDoc(writer, generateStrSeq(0, i + 1));
    }
    IndexReader reader = writer.getReader();
    writer.close();
    // setup MLT query
    MoreLikeThis mlt = new MoreLikeThis(reader);
    mlt.setAnalyzer(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
    mlt.setMaxQueryTerms(topN);
    mlt.setMinDocFreq(1);
    mlt.setMinTermFreq(1);
    mlt.setMinWordLen(1);
    mlt.setFieldNames(new String[] { "text" });
    // perform MLT query
    String likeText = "";
    for (String text : generateStrSeq(0, numDocs)) {
        likeText += text + " ";
    }
    BooleanQuery query = (BooleanQuery) mlt.like("text", new StringReader(likeText));
    // check best terms are topN of highest idf
    List<BooleanClause> clauses = query.clauses();
    assertEquals("Expected" + topN + "clauses only!", topN, clauses.size());
    Term[] expectedTerms = new Term[topN];
    int idx = 0;
    for (String text : generateStrSeq(numDocs - topN, topN)) {
        expectedTerms[idx++] = new Term("text", text);
    }
    for (BooleanClause clause : clauses) {
        Term term = ((TermQuery) clause.getQuery()).getTerm();
        assertTrue(Arrays.asList(expectedTerms).contains(term));
    }
    // clean up
    reader.close();
    dir.close();
}

Example 57 with Directory

use of org.apache.lucene.store.Directory in project elasticsearch by elastic.

the class ShardCoreKeyMapTests method testMissingShard.

public void testMissingShard() throws IOException {
    try (Directory dir = newDirectory();
        RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
        w.addDocument(new Document());
        try (IndexReader reader = w.getReader()) {
            ShardCoreKeyMap map = new ShardCoreKeyMap();
            for (LeafReaderContext ctx : reader.leaves()) {
                try {
                    map.add(ctx.reader());
                    fail();
                } catch (IllegalArgumentException expected) {
                // ok
                }
            }
        }
    }
}

Example 58 with Directory

use of org.apache.lucene.store.Directory in project elasticsearch by elastic.

the class ShardCoreKeyMapTests method testBasics.

public void testBasics() throws IOException {
    Directory dir1 = newDirectory();
    RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1);
    w1.addDocument(new Document());
    Directory dir2 = newDirectory();
    RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2);
    w2.addDocument(new Document());
    Directory dir3 = newDirectory();
    RandomIndexWriter w3 = new RandomIndexWriter(random(), dir3);
    w3.addDocument(new Document());
    ShardId shardId1 = new ShardId("index1", "_na_", 1);
    ShardId shardId2 = new ShardId("index1", "_na_", 3);
    ShardId shardId3 = new ShardId("index2", "_na_", 2);
    ElasticsearchDirectoryReader reader1 = ElasticsearchDirectoryReader.wrap(w1.getReader(), shardId1);
    ElasticsearchDirectoryReader reader2 = ElasticsearchDirectoryReader.wrap(w2.getReader(), shardId2);
    ElasticsearchDirectoryReader reader3 = ElasticsearchDirectoryReader.wrap(w3.getReader(), shardId3);
    ShardCoreKeyMap map = new ShardCoreKeyMap();
    for (DirectoryReader reader : Arrays.asList(reader1, reader2, reader3)) {
        for (LeafReaderContext ctx : reader.leaves()) {
            map.add(ctx.reader());
        }
    }
    assertEquals(3, map.size());
    // Adding them back is a no-op
    for (LeafReaderContext ctx : reader1.leaves()) {
        map.add(ctx.reader());
    }
    assertEquals(3, map.size());
    for (LeafReaderContext ctx : reader2.leaves()) {
        assertEquals(shardId2, map.getShardId(ctx.reader().getCoreCacheKey()));
    }
    w1.addDocument(new Document());
    ElasticsearchDirectoryReader newReader1 = ElasticsearchDirectoryReader.wrap(w1.getReader(), shardId1);
    reader1.close();
    reader1 = newReader1;
    // same for reader2, but with a force merge to trigger evictions
    w2.addDocument(new Document());
    w2.forceMerge(1);
    ElasticsearchDirectoryReader newReader2 = ElasticsearchDirectoryReader.wrap(w2.getReader(), shardId2);
    reader2.close();
    reader2 = newReader2;
    for (DirectoryReader reader : Arrays.asList(reader1, reader2, reader3)) {
        for (LeafReaderContext ctx : reader.leaves()) {
            map.add(ctx.reader());
        }
    }
    final Set<Object> index1Keys = new HashSet<>();
    for (DirectoryReader reader : Arrays.asList(reader1, reader2)) {
        for (LeafReaderContext ctx : reader.leaves()) {
            index1Keys.add(ctx.reader().getCoreCacheKey());
        }
    }
    index1Keys.removeAll(map.getCoreKeysForIndex("index1"));
    assertEquals(Collections.emptySet(), index1Keys);
    reader1.close();
    w1.close();
    reader2.close();
    w2.close();
    reader3.close();
    w3.close();
    assertEquals(0, map.size());
    dir1.close();
    dir2.close();
    dir3.close();
}

Example 59 with Directory

use of org.apache.lucene.store.Directory in project elasticsearch by elastic.

the class SimpleAllTests method testMultipleTokensAllNoBoost.

public void testMultipleTokensAllNoBoost() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    FieldType allFt = getAllFieldType();
    Document doc = new Document();
    doc.add(new Field("_id", "1", StoredField.TYPE));
    doc.add(new AllField("_all", "something moo", 1.0f, allFt));
    doc.add(new AllField("_all", "else koo", 1.0f, allFt));
    indexWriter.addDocument(doc);
    doc = new Document();
    doc.add(new Field("_id", "2", StoredField.TYPE));
    doc.add(new AllField("_all", "else koo", 1.0f, allFt));
    doc.add(new AllField("_all", "something moo", 1.0f, allFt));
    indexWriter.addDocument(doc);
    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    docs = searcher.search(new AllTermQuery(new Term("_all", "koo")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    docs = searcher.search(new AllTermQuery(new Term("_all", "moo")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    indexWriter.close();
}

Example 60 with Directory

use of org.apache.lucene.store.Directory in project elasticsearch by elastic.

the class SimpleAllTests method testMultipleTokensAllWithBoost.

public void testMultipleTokensAllWithBoost() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    FieldType allFt = getAllFieldType();
    Document doc = new Document();
    doc.add(new Field("_id", "1", StoredField.TYPE));
    doc.add(new AllField("_all", "something moo", 1.0f, allFt));
    doc.add(new AllField("_all", "else koo", 1.0f, allFt));
    indexWriter.addDocument(doc);
    doc = new Document();
    doc.add(new Field("_id", "2", StoredField.TYPE));
    doc.add(new AllField("_all", "else koo", 2.0f, allFt));
    doc.add(new AllField("_all", "something moo", 1.0f, allFt));
    indexWriter.addDocument(doc);
    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs docs = searcher.search(new AllTermQuery(new Term("_all", "else")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(1));
    assertThat(docs.scoreDocs[1].doc, equalTo(0));
    docs = searcher.search(new AllTermQuery(new Term("_all", "koo")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(1));
    assertThat(docs.scoreDocs[1].doc, equalTo(0));
    docs = searcher.search(new AllTermQuery(new Term("_all", "something")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    docs = searcher.search(new AllTermQuery(new Term("_all", "moo")), 10);
    assertThat(docs.totalHits, equalTo(2));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
    assertThat(docs.scoreDocs[1].doc, equalTo(1));
    indexWriter.close();
}