Examples with ParsedDocument org.elasticsearch.index.mapper.ParsedDocument used on opensource projects

Example 71 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSearchResultRelease.

public void testSearchResultRelease() throws Exception {
    engine.refresh("warm_up");
    Engine.Searcher searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    searchResult.close();
    // create a document
    ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
    engine.index(indexForDoc(doc));
    // its not there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    // refresh and it should be there
    engine.refresh("test");
    // now its there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    // don't release the search result yet...
    // delete, refresh and do a new search, it should not be there
    engine.delete(new Engine.Delete("1", newUid(doc), UNASSIGNED_SEQ_NO, primaryTerm.get(), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
    engine.refresh("test");
    Engine.Searcher updateSearchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    updateSearchResult.close();
    // the non release search result should not see the deleted yet...
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    searchResult.close();
}

Example 72 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSegmentsWithMergeFlag.

@Test
public void testSegmentsWithMergeFlag() throws Exception {
    try (Store store = createStore();
        Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
        ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
        Engine.Index index = indexForDoc(doc);
        engine.index(index);
        engine.flush();
        assertThat(engine.segments(false).size(), equalTo(1));
        index = indexForDoc(testParsedDocument("2", null, testDocument(), B_1, null));
        engine.index(index);
        engine.flush();
        List<Segment> segments = engine.segments(false);
        assertThat(segments.size(), equalTo(2));
        for (Segment segment : segments) {
            assertThat(segment.getMergeId(), nullValue());
        }
        index = indexForDoc(testParsedDocument("3", null, testDocument(), B_1, null));
        engine.index(index);
        engine.flush();
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(3));
        for (Segment segment : segments) {
            assertThat(segment.getMergeId(), nullValue());
        }
        index = indexForDoc(doc);
        engine.index(index);
        engine.flush();
        final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration();
        // now, optimize and wait for merges, see that we have no merge flag
        engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
        for (Segment segment : engine.segments(false)) {
            assertThat(segment.getMergeId(), nullValue());
        }
        // we could have multiple underlying merges, so the generation may increase more than once
        assertTrue(store.readLastCommittedSegmentsInfo().getGeneration() > gen1);
        final boolean flush = randomBoolean();
        final long gen2 = store.readLastCommittedSegmentsInfo().getGeneration();
        engine.forceMerge(flush, 1, false, false, false, UUIDs.randomBase64UUID());
        for (Segment segment : engine.segments(false)) {
            assertThat(segment.getMergeId(), nullValue());
        }
        if (flush) {
            // we should have had just 1 merge, so last generation should be exact
            assertEquals(gen2, store.readLastCommittedSegmentsInfo().getLastGeneration());
        }
    }
}

Example 73 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testStressUpdateSameDocWhileGettingIt.

@Test
public void testStressUpdateSameDocWhileGettingIt() throws IOException, InterruptedException {
    final int iters = randomIntBetween(1, 15);
    for (int i = 0; i < iters; i++) {
        // this is a reproduction of https://github.com/elastic/elasticsearch/issues/28714
        try (Store store = createStore();
            InternalEngine engine = createEngine(store, createTempDir())) {
            final IndexSettings indexSettings = engine.config().getIndexSettings();
            final IndexMetadata indexMetadata = IndexMetadata.builder(indexSettings.getIndexMetadata()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), TimeValue.timeValueMillis(1))).build();
            engine.engineConfig.getIndexSettings().updateIndexMetadata(indexMetadata);
            engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
            ParsedDocument document = testParsedDocument(Integer.toString(0), null, testDocumentWithTextField(), SOURCE, null);
            final Engine.Index doc = new Engine.Index(newUid(document), document, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
            // first index an append only document and then delete it. such that we have it in the tombstones
            engine.index(doc);
            engine.delete(new Engine.Delete(doc.id(), doc.uid(), UNASSIGNED_SEQ_NO, primaryTerm.get(), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
            // now index more append only docs and refresh so we re-enabel the optimization for unsafe version map
            ParsedDocument document1 = testParsedDocument(Integer.toString(1), null, testDocumentWithTextField(), SOURCE, null);
            engine.index(new Engine.Index(newUid(document1), document1, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0));
            engine.refresh("test");
            ParsedDocument document2 = testParsedDocument(Integer.toString(2), null, testDocumentWithTextField(), SOURCE, null);
            engine.index(new Engine.Index(newUid(document2), document2, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0));
            engine.refresh("test");
            ParsedDocument document3 = testParsedDocument(Integer.toString(3), null, testDocumentWithTextField(), SOURCE, null);
            final Engine.Index doc3 = new Engine.Index(newUid(document3), document3, UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
            engine.index(doc3);
            engine.engineConfig.setEnableGcDeletes(true);
            // once we are here the version map is unsafe again and we need to do a refresh inside the get calls to ensure we
            // de-optimize. We also enabled GCDeletes which now causes pruning tombstones inside that refresh that is done internally
            // to ensure we de-optimize. One get call will purne and the other will try to lock the version map concurrently while
            // holding the lock that pruneTombstones needs and we have a deadlock
            CountDownLatch awaitStarted = new CountDownLatch(1);
            Thread thread = new Thread(() -> {
                awaitStarted.countDown();
                try (Engine.GetResult getResult = engine.get(new Engine.Get(doc3.id(), doc3.uid()), engine::acquireSearcher)) {
                    assertThat(getResult.docIdAndVersion(), is(notNullValue()));
                }
            });
            thread.start();
            awaitStarted.await();
            try (Engine.GetResult getResult = engine.get(new Engine.Get(doc.id(), doc.uid()), engine::acquireSearcher)) {
                assertThat(getResult.docIdAndVersion(), is(nullValue()));
            }
            thread.join();
        }
    }
}

Example 74 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testSegmentsWithoutSoftDeletes.

@Test
public void testSegmentsWithoutSoftDeletes() throws Exception {
    Settings settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build());
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
        List<Segment> segments = engine.segments(false);
        assertThat(segments.isEmpty(), equalTo(true));
        // create two docs and refresh
        ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
        Engine.Index first = indexForDoc(doc);
        Engine.IndexResult firstResult = engine.index(first);
        ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null);
        Engine.Index second = indexForDoc(doc2);
        Engine.IndexResult secondResult = engine.index(second);
        assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation()));
        engine.refresh("test");
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(1));
        assertThat(segments.get(0).isCommitted(), equalTo(false));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(2));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(0).ramTree, nullValue());
        assertThat(segments.get(0).getAttributes().keySet(), Matchers.contains(Lucene87StoredFieldsFormat.MODE_KEY));
        engine.flush();
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(1));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(2));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        ParsedDocument doc3 = testParsedDocument("3", null, testDocumentWithTextField(), B_3, null);
        engine.index(indexForDoc(doc3));
        engine.refresh("test");
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(2));
        assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(2));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(1).isCommitted(), equalTo(false));
        assertThat(segments.get(1).isSearch(), equalTo(true));
        assertThat(segments.get(1).getNumDocs(), equalTo(1));
        assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(1).isCompound(), equalTo(true));
        engine.delete(new Engine.Delete("1", newUid(doc), UNASSIGNED_SEQ_NO, primaryTerm.get(), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
        engine.refresh("test");
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(2));
        assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(1));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(1).isCommitted(), equalTo(false));
        assertThat(segments.get(1).isSearch(), equalTo(true));
        assertThat(segments.get(1).getNumDocs(), equalTo(1));
        assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(1).isCompound(), equalTo(true));
        engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
        ParsedDocument doc4 = testParsedDocument("4", null, testDocumentWithTextField(), B_3, null);
        engine.index(indexForDoc(doc4));
        engine.refresh("test");
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(3));
        assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(1));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(1).isCommitted(), equalTo(false));
        assertThat(segments.get(1).isSearch(), equalTo(true));
        assertThat(segments.get(1).getNumDocs(), equalTo(1));
        assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(1).isCompound(), equalTo(true));
        assertThat(segments.get(2).isCommitted(), equalTo(false));
        assertThat(segments.get(2).isSearch(), equalTo(true));
        assertThat(segments.get(2).getNumDocs(), equalTo(1));
        assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(2).isCompound(), equalTo(true));
        // internal refresh - lets make sure we see those segments in the stats
        ParsedDocument doc5 = testParsedDocument("5", null, testDocumentWithTextField(), B_3, null);
        engine.index(indexForDoc(doc5));
        engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(4));
        assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(1));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(1).isCommitted(), equalTo(false));
        assertThat(segments.get(1).isSearch(), equalTo(true));
        assertThat(segments.get(1).getNumDocs(), equalTo(1));
        assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(1).isCompound(), equalTo(true));
        assertThat(segments.get(2).isCommitted(), equalTo(false));
        assertThat(segments.get(2).isSearch(), equalTo(true));
        assertThat(segments.get(2).getNumDocs(), equalTo(1));
        assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(2).isCompound(), equalTo(true));
        assertThat(segments.get(3).isCommitted(), equalTo(false));
        assertThat(segments.get(3).isSearch(), equalTo(false));
        assertThat(segments.get(3).getNumDocs(), equalTo(1));
        assertThat(segments.get(3).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(3).isCompound(), equalTo(true));
        // now refresh the external searcher and make sure it has the new segment
        engine.refresh("test");
        segments = engine.segments(false);
        assertThat(segments.size(), equalTo(4));
        assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
        assertThat(segments.get(0).isCommitted(), equalTo(true));
        assertThat(segments.get(0).isSearch(), equalTo(true));
        assertThat(segments.get(0).getNumDocs(), equalTo(1));
        assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
        assertThat(segments.get(0).isCompound(), equalTo(true));
        assertThat(segments.get(1).isCommitted(), equalTo(false));
        assertThat(segments.get(1).isSearch(), equalTo(true));
        assertThat(segments.get(1).getNumDocs(), equalTo(1));
        assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(1).isCompound(), equalTo(true));
        assertThat(segments.get(2).isCommitted(), equalTo(false));
        assertThat(segments.get(2).isSearch(), equalTo(true));
        assertThat(segments.get(2).getNumDocs(), equalTo(1));
        assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(2).isCompound(), equalTo(true));
        assertThat(segments.get(3).isCommitted(), equalTo(false));
        assertThat(segments.get(3).isSearch(), equalTo(true));
        assertThat(segments.get(3).getNumDocs(), equalTo(1));
        assertThat(segments.get(3).getDeletedDocs(), equalTo(0));
        assertThat(segments.get(3).isCompound(), equalTo(true));
    }
}

Example 75 with ParsedDocument

use of org.elasticsearch.index.mapper.ParsedDocument in project crate by crate.

the class InternalEngineTests method testRecoveryFromTranslogUpToSeqNo.

@Test
public void testRecoveryFromTranslogUpToSeqNo() throws IOException {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        final long maxSeqNo;
        try (InternalEngine engine = createEngine(config)) {
            final int docs = randomIntBetween(1, 100);
            for (int i = 0; i < docs; i++) {
                final String id = Integer.toString(i);
                final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                engine.index(indexForDoc(doc));
                if (rarely()) {
                    engine.rollTranslogGeneration();
                } else if (rarely()) {
                    engine.flush(randomBoolean(), true);
                }
            }
            maxSeqNo = engine.getLocalCheckpointTracker().getMaxSeqNo();
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getProcessedLocalCheckpoint()));
            engine.syncTranslog();
        }
        try (InternalEngine engine = new InternalEngine(config)) {
            engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
            assertThat(engine.getProcessedLocalCheckpoint(), equalTo(maxSeqNo));
            assertThat(engine.getLocalCheckpointTracker().getMaxSeqNo(), equalTo(maxSeqNo));
        }
        try (InternalEngine engine = new InternalEngine(config)) {
            long upToSeqNo = randomLongBetween(globalCheckpoint.get(), maxSeqNo);
            engine.recoverFromTranslog(translogHandler, upToSeqNo);
            assertThat(engine.getProcessedLocalCheckpoint(), equalTo(upToSeqNo));
            assertThat(engine.getLocalCheckpointTracker().getMaxSeqNo(), equalTo(upToSeqNo));
        }
    }
}