Examples with StringField org.apache.lucene.document.StringField used on opensource projects

Example 51 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestIndexWriterDelete method testFlushPushedDeletesByRAM.

// LUCENE-3340: make sure deletes that we don't apply
// during flush (ie are just pushed into the stream) are
// in fact later flushed due to their RAM usage:
public void testFlushPushedDeletesByRAM() throws Exception {
    Directory dir = newDirectory();
    // Cannot use RandomIndexWriter because we don't want to
    // ever call commit() for this test:
    // note: tiny rambuffer used, as with a 1MB buffer the test is too slow (flush @ 128,999)
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setRAMBufferSizeMB(0.1f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.INSTANCE).setReaderPooling(false));
    int count = 0;
    while (true) {
        Document doc = new Document();
        doc.add(new StringField("id", count + "", Field.Store.NO));
        final Term delTerm;
        if (count == 1010) {
            // This is the only delete that applies
            delTerm = new Term("id", "" + 0);
        } else {
            // These get buffered, taking up RAM, but delete
            // nothing when applied:
            delTerm = new Term("id", "x" + count);
        }
        w.updateDocument(delTerm, doc);
        // TODO: fix this test
        if (slowFileExists(dir, "_0_1.del") || slowFileExists(dir, "_0_1.liv")) {
            if (VERBOSE) {
                System.out.println("TEST: deletes created @ count=" + count);
            }
            break;
        }
        count++;
        // del term we're unlikely to go over 100K:
        if (count > 100000) {
            fail("delete's were not applied");
        }
    }
    w.close();
    dir.close();
}

Example 52 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestPointValues method testDeleteAllPointDocs.

public void testDeleteAllPointDocs() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "0", Field.Store.NO));
    doc.add(new IntPoint("int", 17));
    w.addDocument(doc);
    w.addDocument(new Document());
    w.commit();
    w.deleteDocuments(new Term("id", "0"));
    w.forceMerge(1);
    DirectoryReader r = w.getReader();
    assertNull(r.leaves().get(0).reader().getPointValues("int"));
    w.close();
    r.close();
    dir.close();
}

Example 53 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestLRUQueryCache method testBooleanQueryCachesSubClauses.

public void testBooleanQueryCachesSubClauses() throws IOException {
    Directory dir = newDirectory();
    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    doc.add(new StringField("foo", "bar", Store.YES));
    doc.add(new StringField("foo", "quux", Store.YES));
    w.addDocument(doc);
    w.commit();
    final IndexReader reader = w.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    w.close();
    final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true);
    searcher.setQueryCache(queryCache);
    searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    BooleanQuery.Builder bq = new BooleanQuery.Builder();
    TermQuery should = new TermQuery(new Term("foo", "baz"));
    TermQuery must = new TermQuery(new Term("foo", "bar"));
    TermQuery filter = new TermQuery(new Term("foo", "quux"));
    TermQuery mustNot = new TermQuery(new Term("foo", "foo"));
    bq.add(should, Occur.SHOULD);
    bq.add(must, Occur.MUST);
    bq.add(filter, Occur.FILTER);
    bq.add(mustNot, Occur.MUST_NOT);
    // same bq but with FILTER instead of MUST
    BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
    bq2.add(should, Occur.SHOULD);
    bq2.add(must, Occur.FILTER);
    bq2.add(filter, Occur.FILTER);
    bq2.add(mustNot, Occur.MUST_NOT);
    assertEquals(Collections.emptySet(), new HashSet<>(queryCache.cachedQueries()));
    searcher.search(bq.build(), 1);
    assertEquals(new HashSet<>(Arrays.asList(filter, mustNot)), new HashSet<>(queryCache.cachedQueries()));
    queryCache.clear();
    assertEquals(Collections.emptySet(), new HashSet<>(queryCache.cachedQueries()));
    searcher.search(new ConstantScoreQuery(bq.build()), 1);
    assertEquals(new HashSet<>(Arrays.asList(bq2.build(), should, must, filter, mustNot)), new HashSet<>(queryCache.cachedQueries()));
    reader.close();
    dir.close();
}

Example 54 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestLRUQueryCache method testConcurrency.

public void testConcurrency() throws Throwable {
    final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(20), 1 + random().nextInt(10000), context -> random().nextBoolean());
    Directory dir = newDirectory();
    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    final SearcherFactory searcherFactory = new SearcherFactory() {

        @Override
        public IndexSearcher newSearcher(IndexReader reader, IndexReader previous) throws IOException {
            IndexSearcher searcher = new IndexSearcher(reader);
            searcher.setQueryCachingPolicy(MAYBE_CACHE_POLICY);
            searcher.setQueryCache(queryCache);
            return searcher;
        }
    };
    final boolean applyDeletes = random().nextBoolean();
    final SearcherManager mgr = new SearcherManager(w.w, applyDeletes, false, searcherFactory);
    final AtomicBoolean indexing = new AtomicBoolean(true);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final int numDocs = atLeast(10000);
    Thread[] threads = new Thread[3];
    threads[0] = new Thread() {

        public void run() {
            Document doc = new Document();
            StringField f = new StringField("color", "", Store.NO);
            doc.add(f);
            for (int i = 0; indexing.get() && i < numDocs; ++i) {
                f.setStringValue(RandomPicks.randomFrom(random(), new String[] { "blue", "red", "yellow" }));
                try {
                    w.addDocument(doc);
                    if ((i & 63) == 0) {
                        mgr.maybeRefresh();
                        if (rarely()) {
                            queryCache.clear();
                        }
                        if (rarely()) {
                            final String color = RandomPicks.randomFrom(random(), new String[] { "blue", "red", "yellow" });
                            w.deleteDocuments(new Term("color", color));
                        }
                    }
                } catch (Throwable t) {
                    error.compareAndSet(null, t);
                    break;
                }
            }
            indexing.set(false);
        }
    };
    for (int i = 1; i < threads.length; ++i) {
        threads[i] = new Thread() {

            @Override
            public void run() {
                while (indexing.get()) {
                    try {
                        final IndexSearcher searcher = mgr.acquire();
                        try {
                            final String value = RandomPicks.randomFrom(random(), new String[] { "blue", "red", "yellow", "green" });
                            final Query q = new TermQuery(new Term("color", value));
                            TotalHitCountCollector collector = new TotalHitCountCollector();
                            // will use the cache
                            searcher.search(q, collector);
                            final int totalHits1 = collector.getTotalHits();
                            // will not use the cache because of scores
                            final int totalHits2 = searcher.search(q, 1).totalHits;
                            assertEquals(totalHits2, totalHits1);
                        } finally {
                            mgr.release(searcher);
                        }
                    } catch (Throwable t) {
                        error.compareAndSet(null, t);
                    }
                }
            }
        };
    }
    for (Thread thread : threads) {
        thread.start();
    }
    for (Thread thread : threads) {
        thread.join();
    }
    if (error.get() != null) {
        throw error.get();
    }
    queryCache.assertConsistent();
    mgr.close();
    w.close();
    dir.close();
    queryCache.assertConsistent();
}

Example 55 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestLRUQueryCache method testLRUEviction.

public void testLRUEviction() throws Exception {
    Directory dir = newDirectory();
    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    StringField f = new StringField("color", "blue", Store.NO);
    doc.add(f);
    w.addDocument(doc);
    f.setStringValue("red");
    w.addDocument(doc);
    f.setStringValue("green");
    w.addDocument(doc);
    final DirectoryReader reader = w.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    final LRUQueryCache queryCache = new LRUQueryCache(2, 100000, context -> true);
    final Query blue = new TermQuery(new Term("color", "blue"));
    final Query red = new TermQuery(new Term("color", "red"));
    final Query green = new TermQuery(new Term("color", "green"));
    assertEquals(Collections.emptyList(), queryCache.cachedQueries());
    searcher.setQueryCache(queryCache);
    // the filter is not cached on any segment: no changes
    searcher.setQueryCachingPolicy(NEVER_CACHE);
    searcher.search(new ConstantScoreQuery(green), 1);
    assertEquals(Collections.emptyList(), queryCache.cachedQueries());
    searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    searcher.search(new ConstantScoreQuery(red), 1);
    assertEquals(Collections.singletonList(red), queryCache.cachedQueries());
    searcher.search(new ConstantScoreQuery(green), 1);
    assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
    searcher.search(new ConstantScoreQuery(red), 1);
    assertEquals(Arrays.asList(green, red), queryCache.cachedQueries());
    searcher.search(new ConstantScoreQuery(blue), 1);
    assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
    searcher.search(new ConstantScoreQuery(blue), 1);
    assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
    searcher.search(new ConstantScoreQuery(green), 1);
    assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
    searcher.setQueryCachingPolicy(NEVER_CACHE);
    searcher.search(new ConstantScoreQuery(red), 1);
    assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
    reader.close();
    w.close();
    dir.close();
}