Examples with IdOnlyFieldVisitor org.opensearch.index.fieldvisitor.IdOnlyFieldVisitor used on opensource projects

Example 1 with IdOnlyFieldVisitor

use of org.opensearch.index.fieldvisitor.IdOnlyFieldVisitor in project OpenSearch by opensearch-project.

the class InternalEngine method restoreVersionMapAndCheckpointTracker.

/**
 * Restores the live version map and local checkpoint of this engine using documents (including soft-deleted)
 * after the local checkpoint in the safe commit. This step ensures the live version map and checkpoint tracker
 * are in sync with the Lucene commit.
 */
private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryReader) throws IOException {
    final IndexSearcher searcher = new IndexSearcher(directoryReader);
    searcher.setQueryCache(null);
    final Query query = new BooleanQuery.Builder().add(LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE), BooleanClause.Occur.MUST).add(new DocValuesFieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME), BooleanClause.Occur.MUST).build();
    final Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    for (LeafReaderContext leaf : directoryReader.leaves()) {
        final Scorer scorer = weight.scorer(leaf);
        if (scorer == null) {
            continue;
        }
        final CombinedDocValues dv = new CombinedDocValues(leaf.reader());
        final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor();
        final DocIdSetIterator iterator = scorer.iterator();
        int docId;
        while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
            final long primaryTerm = dv.docPrimaryTerm(docId);
            final long seqNo = dv.docSeqNo(docId);
            localCheckpointTracker.markSeqNoAsProcessed(seqNo);
            localCheckpointTracker.markSeqNoAsPersisted(seqNo);
            idFieldVisitor.reset();
            leaf.reader().document(docId, idFieldVisitor);
            if (idFieldVisitor.getId() == null) {
                assert dv.isTombstone(docId);
                continue;
            }
            final BytesRef uid = new Term(IdFieldMapper.NAME, Uid.encodeId(idFieldVisitor.getId())).bytes();
            try (Releasable ignored = versionMap.acquireLock(uid)) {
                final VersionValue curr = versionMap.getUnderLock(uid);
                if (curr == null || compareOpToVersionMapOnSeqNo(idFieldVisitor.getId(), seqNo, primaryTerm, curr) == OpVsLuceneDocStatus.OP_NEWER) {
                    if (dv.isTombstone(docId)) {
                        // use 0L for the start time so we can prune this delete tombstone quickly
                        // when the local checkpoint advances (i.e., after a recovery completed).
                        final long startTime = 0L;
                        versionMap.putDeleteUnderLock(uid, new DeleteVersionValue(dv.docVersion(docId), seqNo, primaryTerm, startTime));
                    } else {
                        versionMap.putIndexUnderLock(uid, new IndexVersionValue(null, dv.docVersion(docId), seqNo, primaryTerm));
                    }
                }
            }
        }
    }
    // remove live entries in the version map
    refresh("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL, true);
}

Example 2 with IdOnlyFieldVisitor

use of org.opensearch.index.fieldvisitor.IdOnlyFieldVisitor in project OpenSearch by opensearch-project.

the class EngineTestCase method assertAtMostOneLuceneDocumentPerSequenceNumber.

public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings indexSettings, DirectoryReader reader) throws IOException {
    Set<Long> seqNos = new HashSet<>();
    final DirectoryReader wrappedReader = indexSettings.isSoftDeleteEnabled() ? Lucene.wrapAllDocsLive(reader) : reader;
    for (LeafReaderContext leaf : wrappedReader.leaves()) {
        NumericDocValues primaryTermDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME);
        NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
        int docId;
        while ((docId = seqNoDocValues.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
            assertTrue(seqNoDocValues.advanceExact(docId));
            long seqNo = seqNoDocValues.longValue();
            assertThat(seqNo, greaterThanOrEqualTo(0L));
            if (primaryTermDocValues.advanceExact(docId)) {
                if (seqNos.add(seqNo) == false) {
                    final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor();
                    leaf.reader().document(docId, idFieldVisitor);
                    throw new AssertionError("found multiple documents for seq=" + seqNo + " id=" + idFieldVisitor.getId());
                }
            }
        }
    }
}