Examples with Document org.opensearch.index.mapper.ParseContext.Document used on opensource projects

Example 1 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSyncedFlushVanishesOnReplay.

public void testSyncedFlushVanishesOnReplay() throws IOException {
    IOUtils.close(store, engine);
    SetOnce<IndexWriter> indexWriterHolder = new SetOnce<>();
    IndexWriterFactory indexWriterFactory = (directory, iwc) -> {
        indexWriterHolder.set(new IndexWriter(directory, iwc));
        return indexWriterHolder.get();
    };
    store = createStore();
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    engine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), indexWriterFactory, null, globalCheckpoint::get);
    final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
    ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}"), null);
    globalCheckpoint.set(engine.getProcessedLocalCheckpoint());
    engine.index(indexForDoc(doc));
    engine.flush();
    syncFlush(indexWriterHolder.get(), engine, syncId);
    assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
    doc = testParsedDocument("2", null, testDocumentWithTextField(), new BytesArray("{}"), null);
    engine.index(indexForDoc(doc));
    EngineConfig config = engine.config();
    engine.close();
    engine = new InternalEngine(config);
    engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
    assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID));
}

Example 2 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSimpleOperations.

public void testSimpleOperations() throws Exception {
    engine.refresh("warm_up");
    Engine.Searcher searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    searchResult.close();
    final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
    // create a document
    Document document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    ParsedDocument doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(indexForDoc(doc));
    // its not there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    searchResult.close();
    // but, not there non realtime
    try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(false));
    }
    // but, we can still get it (in realtime)
    try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(true));
        assertThat(getResult.docIdAndVersion(), notNullValue());
    }
    // but not real time is not yet visible
    try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(false));
    }
    // refresh and it should be there
    engine.refresh("test");
    // now its there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    searchResult.close();
    // also in non realtime
    try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(true));
        assertThat(getResult.docIdAndVersion(), notNullValue());
    }
    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", null, document, B_2, null);
    engine.index(indexForDoc(doc));
    // its not updated yet...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // but, we can still get it (in realtime)
    try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(true));
        assertThat(getResult.docIdAndVersion(), notNullValue());
    }
    // refresh and it should be updated
    engine.refresh("test");
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // now delete
    engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get()));
    // its not deleted yet
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
    // but, get should not see it (in realtime)
    try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(false));
    }
    // refresh and it should be deleted
    engine.refresh("test");
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // add it back
    document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED));
    // its not there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // refresh and it should be there
    engine.refresh("test");
    // now its there...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // now flush
    engine.flush();
    // and, verify get (in real time)
    try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
        assertThat(getResult.exists(), equalTo(true));
        assertThat(getResult.docIdAndVersion(), notNullValue());
    }
    // make sure we can still work with the engine
    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(indexForDoc(doc));
    // its not updated yet...
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
    searchResult.close();
    // refresh and it should be updated
    engine.refresh("test");
    searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1));
    searchResult.close();
}

Example 3 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testStoreHonorsLuceneVersion.

public void testStoreHonorsLuceneVersion() throws IOException {
    for (Version createdVersion : Arrays.asList(Version.CURRENT, VersionUtils.getPreviousMinorVersion(), VersionUtils.getFirstVersion())) {
        Settings settings = Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion).build();
        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
        try (Store store = createStore();
            InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
            ParsedDocument doc = testParsedDocument("1", null, new Document(), new BytesArray("{}".getBytes("UTF-8")), null);
            engine.index(appendOnlyPrimary(doc, false, 1));
            engine.refresh("test");
            try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
                LeafReader leafReader = getOnlyLeafReader(searcher.getIndexReader());
                assertEquals(createdVersion.luceneVersion.major, leafReader.getMetaData().getCreatedVersionMajor());
            }
        }
    }
}

Example 4 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testTrimUnsafeCommits.

public void testTrimUnsafeCommits() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final int maxSeqNo = 40;
    final List<Long> seqNos = LongStream.rangeClosed(0, maxSeqNo).boxed().collect(Collectors.toList());
    Collections.shuffle(seqNos, random());
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        final List<Long> commitMaxSeqNo = new ArrayList<>();
        final long minTranslogGen;
        try (InternalEngine engine = createEngine(config)) {
            for (int i = 0; i < seqNos.size(); i++) {
                ParsedDocument doc = testParsedDocument(Long.toString(seqNos.get(i)), null, testDocument(), new BytesArray("{}"), null);
                Engine.Index index = new Engine.Index(newUid(doc), doc, seqNos.get(i), 0, 1, null, REPLICA, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
                engine.index(index);
                if (randomBoolean()) {
                    engine.flush();
                    final Long maxSeqNoInCommit = seqNos.subList(0, i + 1).stream().max(Long::compareTo).orElse(-1L);
                    commitMaxSeqNo.add(maxSeqNoInCommit);
                }
            }
            globalCheckpoint.set(randomInt(maxSeqNo));
            engine.syncTranslog();
            minTranslogGen = engine.getTranslog().getMinFileGeneration();
        }
        store.trimUnsafeCommits(globalCheckpoint.get(), minTranslogGen, config.getIndexSettings().getIndexVersionCreated());
        long safeMaxSeqNo = commitMaxSeqNo.stream().filter(s -> s <= globalCheckpoint.get()).reduce(// get the last one.
        (s1, s2) -> s2).orElse(SequenceNumbers.NO_OPS_PERFORMED);
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        assertThat(commits, hasSize(1));
        assertThat(commits.get(0).getUserData().get(SequenceNumbers.MAX_SEQ_NO), equalTo(Long.toString(safeMaxSeqNo)));
        try (IndexReader reader = DirectoryReader.open(commits.get(0))) {
            for (LeafReaderContext context : reader.leaves()) {
                final NumericDocValues values = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
                if (values != null) {
                    for (int docID = 0; docID < context.reader().maxDoc(); docID++) {
                        if (values.advanceExact(docID) == false) {
                            throw new AssertionError("Document does not have a seq number: " + docID);
                        }
                        assertThat(values.longValue(), lessThanOrEqualTo(globalCheckpoint.get()));
                    }
                }
            }
        }
    }
}

Example 5 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testOutOfOrderSequenceNumbersWithVersionConflict.

/**
 * java docs
 */
public void testOutOfOrderSequenceNumbersWithVersionConflict() throws IOException {
    final List<Engine.Operation> operations = new ArrayList<>();
    final int numberOfOperations = randomIntBetween(16, 32);
    final AtomicLong sequenceNumber = new AtomicLong();
    final Engine.Operation.Origin origin = randomFrom(LOCAL_TRANSLOG_RECOVERY, PEER_RECOVERY, PRIMARY, REPLICA);
    final LongSupplier sequenceNumberSupplier = origin == PRIMARY ? () -> UNASSIGNED_SEQ_NO : sequenceNumber::getAndIncrement;
    final Supplier<ParsedDocument> doc = () -> {
        final Document document = testDocumentWithTextField();
        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
        return testParsedDocument("1", null, document, B_1, null);
    };
    final Term uid = newUid("1");
    final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
    for (int i = 0; i < numberOfOperations; i++) {
        if (randomBoolean()) {
            final Engine.Index index = new Engine.Index(uid, doc.get(), sequenceNumberSupplier.getAsLong(), 1, i, origin == PRIMARY ? VersionType.EXTERNAL : null, origin, System.nanoTime(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, UNASSIGNED_SEQ_NO, 0);
            operations.add(index);
        } else {
            final Engine.Delete delete = new Engine.Delete("test", "1", uid, sequenceNumberSupplier.getAsLong(), 1, i, origin == PRIMARY ? VersionType.EXTERNAL : null, origin, System.nanoTime(), UNASSIGNED_SEQ_NO, 0);
            operations.add(delete);
        }
    }
    final boolean exists = operations.get(operations.size() - 1) instanceof Engine.Index;
    Randomness.shuffle(operations);
    for (final Engine.Operation operation : operations) {
        if (operation instanceof Engine.Index) {
            engine.index((Engine.Index) operation);
        } else {
            engine.delete((Engine.Delete) operation);
        }
    }
    final long expectedLocalCheckpoint;
    if (origin == PRIMARY) {
        // we can only advance as far as the number of operations that did not conflict
        int count = 0;
        // each time the version increments as we walk the list, that counts as a successful operation
        long version = -1;
        for (int i = 0; i < numberOfOperations; i++) {
            if (operations.get(i).version() >= version) {
                count++;
                version = operations.get(i).version();
            }
        }
        // sequence numbers start at zero, so the expected local checkpoint is the number of successful operations minus one
        expectedLocalCheckpoint = count - 1;
    } else {
        expectedLocalCheckpoint = numberOfOperations - 1;
    }
    assertThat(engine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
    try (Engine.GetResult result = engine.get(new Engine.Get(true, false, "2", uid), searcherFactory)) {
        assertThat(result.exists(), equalTo(exists));
    }
}