Search in sources :

Example 16 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project elasticsearch by elastic.

the class ShadowEngineTests method testSimpleOperations.

public void testSimpleOperations() throws Exception {
    Engine.Searcher searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    searchResult.close();
    // create a document
    ParseContext.Document document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
    primaryEngine.index(indexForDoc(doc));
    // its not there...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    // not on the replica either...
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    // but, we can still get it (in realtime)
    Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());
    getResult.release();
    // can't get it from the replica, because it's not in the translog for a shadow replica
    getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(false));
    getResult.release();
    // but, not there non realtime
    getResult = primaryEngine.get(new Engine.Get(false, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    getResult.release();
    // now its there...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    searchResult.close();
    // also in non realtime
    getResult = primaryEngine.get(new Engine.Get(false, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());
    getResult.release();
    // still not in the replica because no flush
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", "test", null, document, B_2, null);
    primaryEngine.index(indexForDoc(doc));
    // its not updated yet...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // but, we can still get it (in realtime)
    getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());
    getResult.release();
    // refresh and it should be updated
    primaryEngine.refresh("test");
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // flush, now shadow replica should have the files
    primaryEngine.flush();
    // still not in the replica because the replica hasn't refreshed
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    replicaEngine.refresh("test");
    // the replica finally sees it because primary has flushed and replica refreshed
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // now delete
    primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
    // its not deleted yet
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // but, get should not see it (in realtime)
    getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(false));
    getResult.release();
    // refresh and it should be deleted
    primaryEngine.refresh("test");
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // add it back
    document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", "test", null, document, B_1, null);
    primaryEngine.index(indexForDoc(doc));
    // its not there...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // refresh and it should be there
    primaryEngine.refresh("test");
    // now its there...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // now flush
    primaryEngine.flush();
    // and, verify get (in real time)
    getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());
    getResult.release();
    // the replica should see it if we refresh too!
    replicaEngine.refresh("test");
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());
    getResult.release();
    // make sure we can still work with the engine
    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    doc = testParsedDocument("1", "test", null, document, B_1, null);
    primaryEngine.index(indexForDoc(doc));
    // its not updated yet...
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // refresh and it should be updated
    primaryEngine.refresh("test");
    searchResult = primaryEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // Make visible to shadow replica
    primaryEngine.flush();
    replicaEngine.refresh("test");
    searchResult = replicaEngine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
}
Also used : TermQuery(org.apache.lucene.search.TermQuery) Term(org.apache.lucene.index.Term) NumericDocValuesField(org.apache.lucene.document.NumericDocValuesField) Field(org.apache.lucene.document.Field) TextField(org.apache.lucene.document.TextField) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) ParseContext(org.elasticsearch.index.mapper.ParseContext) TextField(org.apache.lucene.document.TextField)

Example 17 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project elasticsearch by elastic.

the class InternalEngineTests method testRecoverFromForeignTranslog.

public void testRecoverFromForeignTranslog() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult index = engine.index(firstIndexRequest);
        assertThat(index.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    Translog.TranslogGeneration generation = engine.getTranslog().getGeneration();
    engine.close();
    Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO);
    translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8"))));
    assertEquals(generation.translogFileGeneration, translog.currentFileGeneration());
    translog.close();
    EngineConfig config = engine.config();
    /* create a TranslogConfig that has been created with a different UUID */
    TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE);
    EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), config.getRefreshListeners(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    try {
        InternalEngine internalEngine = new InternalEngine(brokenConfig);
        fail("translog belongs to a different engine");
    } catch (EngineCreationFailureException ex) {
    }
    // and recover again!
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}
Also used : Searcher(org.elasticsearch.index.engine.Engine.Searcher) BytesArray(org.elasticsearch.common.bytes.BytesArray) TranslogConfig(org.elasticsearch.index.translog.TranslogConfig) Index(org.elasticsearch.index.Index) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) LongPoint(org.apache.lucene.document.LongPoint) Translog(org.elasticsearch.index.translog.Translog) TopDocs(org.apache.lucene.search.TopDocs) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) CodecService(org.elasticsearch.index.codec.CodecService)

Example 18 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project elasticsearch by elastic.

the class InternalEngineTests method testTranslogReplay.

public void testTranslogReplay() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult indexResult = engine.index(firstIndexRequest);
        assertThat(indexResult.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    TranslogHandler parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
    parser.mappingUpdate = dynamicUpdate();
    engine.close();
    // we need to reuse the engine config unless the parser.mappingModified won't work
    engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
    engine.recoverFromTranslog();
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
    assertEquals(numDocs, parser.recoveredOps.get());
    if (parser.mappingUpdate != null) {
        assertEquals(1, parser.getRecoveredTypes().size());
        assertTrue(parser.getRecoveredTypes().containsKey("test"));
    } else {
        assertEquals(0, parser.getRecoveredTypes().size());
    }
    engine.close();
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
    assertEquals(0, parser.recoveredOps.get());
    final boolean flush = randomBoolean();
    int randomId = randomIntBetween(numDocs + 1, numDocs + 10);
    ParsedDocument doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
    Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
    Engine.IndexResult indexResult = engine.index(firstIndexRequest);
    assertThat(indexResult.getVersion(), equalTo(1L));
    if (flush) {
        engine.flush();
    }
    doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
    Engine.Index idxRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
    Engine.IndexResult result = engine.index(idxRequest);
    engine.refresh("test");
    assertThat(result.getVersion(), equalTo(2L));
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs + 1);
        assertThat(topDocs.totalHits, equalTo(numDocs + 1));
    }
    engine.close();
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs + 1);
        assertThat(topDocs.totalHits, equalTo(numDocs + 1));
    }
    parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
    assertEquals(flush ? 1 : 2, parser.recoveredOps.get());
    engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc)));
    if (randomBoolean()) {
        engine.refresh("test");
    } else {
        engine.close();
        engine = createEngine(store, primaryTranslogDir);
    }
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs);
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}
Also used : Searcher(org.elasticsearch.index.engine.Engine.Searcher) BytesArray(org.elasticsearch.common.bytes.BytesArray) Index(org.elasticsearch.index.Index) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) LongPoint(org.apache.lucene.document.LongPoint) TopDocs(org.apache.lucene.search.TopDocs) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument)

Example 19 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project elasticsearch by elastic.

the class InternalEngineTests method testVersioningReplicaConflict1.

public void testVersioningReplicaConflict1() throws IOException {
    final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
    final Engine.Index v1Index = indexForDoc(doc);
    final Engine.IndexResult v1Result = engine.index(v1Index);
    assertThat(v1Result.getVersion(), equalTo(1L));
    final Engine.Index v2Index = indexForDoc(doc);
    final Engine.IndexResult v2Result = engine.index(v2Index);
    assertThat(v2Result.getVersion(), equalTo(2L));
    // apply the second index to the replica, should work fine
    final Engine.Index replicaV2Index = new Engine.Index(newUid(doc), doc, v2Result.getSeqNo(), v2Index.primaryTerm(), v2Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    final Engine.IndexResult replicaV2Result = replicaEngine.index(replicaV2Index);
    assertThat(replicaV2Result.getVersion(), equalTo(2L));
    // now, the old one should produce an indexing result
    final Engine.Index replicaV1Index = new Engine.Index(newUid(doc), doc, v1Result.getSeqNo(), v1Index.primaryTerm(), v1Result.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
    final Engine.IndexResult replicaV1Result = replicaEngine.index(replicaV1Index);
    assertFalse(replicaV1Result.hasFailure());
    assertFalse(replicaV1Result.isCreated());
    assertThat(replicaV1Result.getVersion(), equalTo(2L));
    // second version on replica should fail as well
    final Engine.IndexResult replicaV2ReplayResult = replicaEngine.index(replicaV2Index);
    assertFalse(replicaV2Result.hasFailure());
    assertFalse(replicaV1Result.isCreated());
    assertThat(replicaV2ReplayResult.getVersion(), equalTo(2L));
}
Also used : ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) Index(org.elasticsearch.index.Index)

Example 20 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project elasticsearch by elastic.

the class InternalEngineTests method testAppendConcurrently.

public void testAppendConcurrently() throws InterruptedException, IOException {
    Thread[] thread = new Thread[randomIntBetween(3, 5)];
    int numDocs = randomIntBetween(1000, 10000);
    assertEquals(0, engine.getNumVersionLookups());
    assertEquals(0, engine.getNumIndexVersionsLookups());
    List<Engine.Index> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
        final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
        Engine.Index index = randomAppendOnly(doc, false, i);
        docs.add(index);
    }
    Collections.shuffle(docs, random());
    CountDownLatch startGun = new CountDownLatch(thread.length);
    AtomicInteger offset = new AtomicInteger(-1);
    for (int i = 0; i < thread.length; i++) {
        thread[i] = new Thread() {

            @Override
            public void run() {
                startGun.countDown();
                try {
                    startGun.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                int docOffset;
                while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                    try {
                        engine.index(docs.get(docOffset));
                    } catch (IOException e) {
                        throw new AssertionError(e);
                    }
                }
            }
        };
        thread[i].start();
    }
    for (int i = 0; i < thread.length; i++) {
        thread[i].join();
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
        assertEquals(docs.size(), topDocs.totalHits);
    }
    assertEquals(0, engine.getNumVersionLookups());
    assertEquals(0, engine.getNumIndexVersionsLookups());
    assertFalse(engine.indexWriterHasDeletions());
}
Also used : Searcher(org.elasticsearch.index.engine.Engine.Searcher) BytesArray(org.elasticsearch.common.bytes.BytesArray) ArrayList(java.util.ArrayList) Index(org.elasticsearch.index.Index) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) CountDownLatch(java.util.concurrent.CountDownLatch) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) LongPoint(org.apache.lucene.document.LongPoint) TopDocs(org.apache.lucene.search.TopDocs) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) AtomicInteger(java.util.concurrent.atomic.AtomicInteger)

Aggregations

ParsedDocument (org.elasticsearch.index.mapper.ParsedDocument)211 Test (org.junit.Test)85 LongPoint (org.apache.lucene.document.LongPoint)59 BytesArray (org.elasticsearch.common.bytes.BytesArray)58 Matchers.containsString (org.hamcrest.Matchers.containsString)57 Store (org.elasticsearch.index.store.Store)52 Searcher (org.elasticsearch.index.engine.Engine.Searcher)46 DocumentMapper (org.elasticsearch.index.mapper.DocumentMapper)35 IOException (java.io.IOException)32 AtomicLong (java.util.concurrent.atomic.AtomicLong)31 MatchAllDocsQuery (org.apache.lucene.search.MatchAllDocsQuery)31 IndexableField (org.apache.lucene.index.IndexableField)30 Term (org.apache.lucene.index.Term)28 TopDocs (org.apache.lucene.search.TopDocs)28 NumericDocValuesField (org.apache.lucene.document.NumericDocValuesField)27 Index (org.elasticsearch.index.Index)27 UncheckedIOException (java.io.UncheckedIOException)26 Field (org.apache.lucene.document.Field)26 TextField (org.apache.lucene.document.TextField)26 ArrayList (java.util.ArrayList)25