Examples with Document org.opensearch.index.mapper.ParseContext.Document used on opensource projects

Example 31 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSequenceIDs.

public void testSequenceIDs() throws Exception {
    Tuple<Long, Long> seqID = getSequenceID(engine, new Engine.Get(false, false, "2", newUid("1")));
    // Non-existent doc returns no seqnum and no primary term
    assertThat(seqID.v1(), equalTo(UNASSIGNED_SEQ_NO));
    assertThat(seqID.v2(), equalTo(0L));
    // create a document
    Document document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    ParsedDocument doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(indexForDoc(doc));
    engine.refresh("test");
    seqID = getSequenceID(engine, newGet(false, doc));
    logger.info("--> got seqID: {}", seqID);
    assertThat(seqID.v1(), equalTo(0L));
    assertThat(seqID.v2(), equalTo(primaryTerm.get()));
    // Index the same document again
    document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(indexForDoc(doc));
    engine.refresh("test");
    seqID = getSequenceID(engine, newGet(false, doc));
    logger.info("--> got seqID: {}", seqID);
    assertThat(seqID.v1(), equalTo(1L));
    assertThat(seqID.v2(), equalTo(primaryTerm.get()));
    // Index the same document for the third time, this time changing the primary term
    document = testDocumentWithTextField();
    document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc = testParsedDocument("1", null, document, B_1, null);
    engine.index(new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, 3, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0));
    engine.refresh("test");
    seqID = getSequenceID(engine, newGet(false, doc));
    logger.info("--> got seqID: {}", seqID);
    assertThat(seqID.v1(), equalTo(2L));
    assertThat(seqID.v2(), equalTo(3L));
    // we can query by the _seq_no
    Engine.Searcher searchResult = engine.acquireSearcher("test");
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(LongPoint.newExactQuery("_seq_no", 2), 1));
    searchResult.close();
}

Example 32 with Document

use of org.opensearch.index.mapper.ParseContext.Document in project OpenSearch by opensearch-project.

the class InternalEngineTests method testKeepTranslogAfterGlobalCheckpoint.

public void testKeepTranslogAfterGlobalCheckpoint() throws Exception {
    IOUtils.close(engine, store);
    final IndexSettings indexSettings = new IndexSettings(defaultSettings.getIndexMetadata(), defaultSettings.getNodeSettings(), defaultSettings.getScopedSettings());
    IndexMetadata.Builder builder = IndexMetadata.builder(indexSettings.getIndexMetadata()).settings(Settings.builder().put(indexSettings.getSettings()).put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomFrom("-1", "100micros", "30m")).put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomFrom("-1", "512b", "1gb")));
    indexSettings.updateIndexMetadata(builder.build());
    final Path translogPath = createTempDir();
    store = createStore();
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    store.createEmpty(Version.CURRENT.luceneVersion);
    final String translogUUID = Translog.createEmptyTranslog(translogPath, globalCheckpoint.get(), shardId, primaryTerm.get());
    store.associateIndexWithNewTranslog(translogUUID);
    final EngineConfig engineConfig = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, () -> globalCheckpoint.get());
    final AtomicLong lastSyncedGlobalCheckpointBeforeCommit = new AtomicLong(Translog.readGlobalCheckpoint(translogPath, translogUUID));
    try (InternalEngine engine = new InternalEngine(engineConfig) {

        @Override
        protected void commitIndexWriter(IndexWriter writer, Translog translog) throws IOException {
            lastSyncedGlobalCheckpointBeforeCommit.set(Translog.readGlobalCheckpoint(translogPath, translogUUID));
            // (this value is visible to the deletion policy) and an in memory global checkpoint in the SequenceNumbersService.
            if (rarely()) {
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), getPersistedLocalCheckpoint()));
            }
            super.commitIndexWriter(writer, translog);
        }
    }) {
        engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        int numDocs = scaledRandomIntBetween(10, 100);
        for (int docId = 0; docId < numDocs; docId++) {
            ParseContext.Document document = testDocumentWithTextField();
            document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
            engine.index(indexForDoc(testParsedDocument(Integer.toString(docId), null, document, B_1, null)));
            if (frequently()) {
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                engine.syncTranslog();
            }
            if (frequently()) {
                engine.flush(randomBoolean(), true);
                final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
                // Keep only one safe commit as the oldest commit.
                final IndexCommit safeCommit = commits.get(0);
                if (lastSyncedGlobalCheckpointBeforeCommit.get() == UNASSIGNED_SEQ_NO) {
                    // If the global checkpoint is still unassigned, we keep an empty(eg. initial) commit as a safe commit.
                    assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
                } else {
                    assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), lessThanOrEqualTo(lastSyncedGlobalCheckpointBeforeCommit.get()));
                }
                for (int i = 1; i < commits.size(); i++) {
                    assertThat(Long.parseLong(commits.get(i).getUserData().get(SequenceNumbers.MAX_SEQ_NO)), greaterThan(lastSyncedGlobalCheckpointBeforeCommit.get()));
                }
                // Make sure we keep all translog operations after the local checkpoint of the safe commit.
                long localCheckpointFromSafeCommit = Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
                try (Translog.Snapshot snapshot = getTranslog(engine).newSnapshot()) {
                    assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(localCheckpointFromSafeCommit + 1, docId));
                }
            }
        }
    }
}