Examples with IndexWriterConfig org.apache.lucene.index.IndexWriterConfig used on opensource projects

Example 31 with IndexWriterConfig

use of org.apache.lucene.index.IndexWriterConfig in project elasticsearch by elastic.

the class ScaledFloatFieldTypeTests method testFieldData.

public void testFieldData() throws IOException {
    ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType();
    ft.setScalingFactor(0.1 + randomDouble() * 100);
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    Document doc = new Document();
    doc.add(new SortedNumericDocValuesField("scaled_float1", 10));
    doc.add(new SortedNumericDocValuesField("scaled_float2", 5));
    doc.add(new SortedNumericDocValuesField("scaled_float2", 12));
    w.addDocument(doc);
    try (DirectoryReader reader = DirectoryReader.open(w)) {
        IndexMetaData indexMetadata = new IndexMetaData.Builder("index").settings(Settings.builder().put("index.version.created", Version.CURRENT).put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()).build();
        IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
        // single-valued
        ft.setName("scaled_float1");
        IndexNumericFieldData fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null);
        assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE);
        AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
        SortedNumericDoubleValues values = leafFieldData.getDoubleValues();
        values.setDocument(0);
        assertEquals(1, values.count());
        assertEquals(10 / ft.getScalingFactor(), values.valueAt(0), 10e-5);
        // multi-valued
        ft.setName("scaled_float2");
        fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null);
        leafFieldData = fielddata.load(reader.leaves().get(0));
        values = leafFieldData.getDoubleValues();
        values.setDocument(0);
        assertEquals(2, values.count());
        assertEquals(5 / ft.getScalingFactor(), values.valueAt(0), 10e-5);
        assertEquals(12 / ft.getScalingFactor(), values.valueAt(1), 10e-5);
    }
    IOUtils.close(w, dir);
}

Example 32 with IndexWriterConfig

use of org.apache.lucene.index.IndexWriterConfig in project elasticsearch by elastic.

the class StoreTests method testCanOpenIndex.

public void testCanOpenIndex() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    IndexWriterConfig iwc = newIndexWriterConfig();
    Path tempDir = createTempDir();
    final BaseDirectoryWrapper dir = newFSDirectory(tempDir);
    assertFalse(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    writer.close();
    assertTrue(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));
    DirectoryService directoryService = new DirectoryService(shardId, INDEX_SETTINGS) {

        @Override
        public Directory newDirectory() throws IOException {
            return dir;
        }
    };
    Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId));
    store.markStoreCorrupted(new CorruptIndexException("foo", "bar"));
    assertFalse(Store.canOpenIndex(logger, tempDir, shardId, (id, l) -> new DummyShardLock(id)));
    store.close();
}

Example 33 with IndexWriterConfig

use of org.apache.lucene.index.IndexWriterConfig in project elasticsearch by elastic.

the class StoreTests method testUserDataRead.

public void testUserDataRead() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId));
    IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec());
    SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    config.setIndexDeletionPolicy(deletionPolicy);
    IndexWriter writer = new IndexWriter(store.directory(), config);
    Document doc = new Document();
    doc.add(new TextField("id", "1", Field.Store.NO));
    writer.addDocument(doc);
    Map<String, String> commitData = new HashMap<>(2);
    String syncId = "a sync id";
    String translogId = "a translog id";
    commitData.put(Engine.SYNC_COMMIT_ID, syncId);
    commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogId);
    writer.setCommitData(commitData);
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata;
    metadata = store.getMetadata(randomBoolean() ? null : deletionPolicy.snapshot());
    assertFalse(metadata.asMap().isEmpty());
    // do not check for correct files, we have enough tests for that above
    assertThat(metadata.getCommitUserData().get(Engine.SYNC_COMMIT_ID), equalTo(syncId));
    assertThat(metadata.getCommitUserData().get(Translog.TRANSLOG_GENERATION_KEY), equalTo(translogId));
    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

Example 34 with IndexWriterConfig

use of org.apache.lucene.index.IndexWriterConfig in project elasticsearch by elastic.

the class IndexSearcherWrapperTests method testReaderCloseListenerIsCalled.

public void testReaderCloseListenerIsCalled() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
    IndexSearcher searcher = new IndexSearcher(open);
    assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits);
    final AtomicInteger closeCalls = new AtomicInteger(0);
    IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {

        @Override
        public DirectoryReader wrap(DirectoryReader reader) throws IOException {
            return new FieldMaskingReader("field", reader, closeCalls);
        }

        @Override
        public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
            return searcher;
        }
    };
    final int sourceRefCount = open.getRefCount();
    final AtomicInteger count = new AtomicInteger();
    final AtomicInteger outerCount = new AtomicInteger();
    try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) {
        final Engine.Searcher wrap = wrapper.wrap(engineSearcher);
        assertEquals(1, wrap.reader().getRefCount());
        ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> {
            if (reader == open) {
                count.incrementAndGet();
            }
            outerCount.incrementAndGet();
        });
        assertEquals(0, wrap.searcher().search(new TermQuery(new Term("field", "doc")), 1).totalHits);
        wrap.close();
        assertFalse("wrapped reader is closed", wrap.reader().tryIncRef());
        assertEquals(sourceRefCount, open.getRefCount());
    }
    assertEquals(1, closeCalls.get());
    IOUtils.close(open, writer, dir);
    assertEquals(1, outerCount.get());
    assertEquals(1, count.get());
    assertEquals(0, open.getRefCount());
    assertEquals(1, closeCalls.get());
}

Example 35 with IndexWriterConfig

use of org.apache.lucene.index.IndexWriterConfig in project elasticsearch by elastic.

the class IndexSearcherWrapperTests method testIsCacheable.

public void testIsCacheable() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
    IndexSearcher searcher = new IndexSearcher(open);
    assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits);
    searcher.setSimilarity(iwc.getSimilarity());
    final AtomicInteger closeCalls = new AtomicInteger(0);
    IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {

        @Override
        public DirectoryReader wrap(DirectoryReader reader) throws IOException {
            return new FieldMaskingReader("field", reader, closeCalls);
        }

        @Override
        public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
            return searcher;
        }
    };
    final ConcurrentHashMap<Object, TopDocs> cache = new ConcurrentHashMap<>();
    try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) {
        try (Engine.Searcher wrap = wrapper.wrap(engineSearcher)) {
            ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> {
                cache.remove(reader.getCoreCacheKey());
            });
            TopDocs search = wrap.searcher().search(new TermQuery(new Term("field", "doc")), 1);
            cache.put(wrap.reader().getCoreCacheKey(), search);
        }
    }
    assertEquals(1, closeCalls.get());
    assertEquals(1, cache.size());
    IOUtils.close(open, writer, dir);
    assertEquals(0, cache.size());
    assertEquals(1, closeCalls.get());
}