Examples with DocIdAndSeqNo org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo used on opensource projects

Example 1 with DocIdAndSeqNo

use of org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo in project OpenSearch by opensearch-project.

the class InternalEngineTests method getSequenceID.

/**
 * Return a tuple representing the sequence ID for the given {@code Get}
 * operation. The first value in the tuple is the sequence number, the
 * second is the primary term.
 */
private Tuple<Long, Long> getSequenceID(Engine engine, Engine.Get get) throws EngineException {
    try (Engine.Searcher searcher = engine.acquireSearcher("get", Engine.SearcherScope.INTERNAL)) {
        final long primaryTerm;
        final long seqNo;
        DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), get.uid());
        if (docIdAndSeqNo == null) {
            primaryTerm = UNASSIGNED_PRIMARY_TERM;
            seqNo = UNASSIGNED_SEQ_NO;
        } else {
            seqNo = docIdAndSeqNo.seqNo;
            NumericDocValues primaryTerms = docIdAndSeqNo.context.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME);
            if (primaryTerms == null || primaryTerms.advanceExact(docIdAndSeqNo.docId) == false) {
                throw new AssertionError("document does not have primary term [" + docIdAndSeqNo.docId + "]");
            }
            primaryTerm = primaryTerms.longValue();
        }
        return new Tuple<>(seqNo, primaryTerm);
    } catch (Exception e) {
        throw new EngineException(shardId, "unable to retrieve sequence id", e);
    }
}

Example 2 with DocIdAndSeqNo

use of org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo in project OpenSearch by opensearch-project.

the class InternalEngine method compareOpToLuceneDocBasedOnSeqNo.

private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException {
    assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found";
    final OpVsLuceneDocStatus status;
    VersionValue versionValue = getVersionFromMap(op.uid().bytes());
    assert incrementVersionLookup();
    if (versionValue != null) {
        status = compareOpToVersionMapOnSeqNo(op.id(), op.seqNo(), op.primaryTerm(), versionValue);
    } else {
        // load from index
        assert incrementIndexVersionLookup();
        try (Searcher searcher = acquireSearcher("load_seq_no", SearcherScope.INTERNAL)) {
            final DocIdAndSeqNo docAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), op.uid());
            if (docAndSeqNo == null) {
                status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
            } else if (op.seqNo() > docAndSeqNo.seqNo) {
                status = OpVsLuceneDocStatus.OP_NEWER;
            } else if (op.seqNo() == docAndSeqNo.seqNo) {
                assert localCheckpointTracker.hasProcessed(op.seqNo()) : "local checkpoint tracker is not updated seq_no=" + op.seqNo() + " id=" + op.id();
                status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
            } else {
                status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
            }
        }
    }
    return status;
}

Example 3 with DocIdAndSeqNo

use of org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo in project OpenSearch by opensearch-project.

the class InternalEngineTests method testLookupSeqNoByIdInLucene.

public void testLookupSeqNoByIdInLucene() throws Exception {
    int numOps = between(10, 100);
    long seqNo = 0;
    List<Engine.Operation> operations = new ArrayList<>(numOps);
    for (int i = 0; i < numOps; i++) {
        String id = Integer.toString(between(1, 50));
        boolean isIndexing = randomBoolean();
        int copies = frequently() ? 1 : between(2, 4);
        for (int c = 0; c < copies; c++) {
            final ParsedDocument doc = EngineTestCase.createParsedDoc(id, null);
            if (isIndexing) {
                operations.add(new Engine.Index(EngineTestCase.newUid(doc), doc, seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), -1, true, UNASSIGNED_SEQ_NO, 0L));
            } else {
                operations.add(new Engine.Delete(doc.type(), doc.id(), EngineTestCase.newUid(doc), seqNo, primaryTerm.get(), i, null, Engine.Operation.Origin.REPLICA, threadPool.relativeTimeInMillis(), UNASSIGNED_SEQ_NO, 0L));
            }
        }
        seqNo++;
        if (rarely()) {
            seqNo++;
        }
    }
    Randomness.shuffle(operations);
    // id -> latest seq_no
    Map<String, Engine.Operation> latestOps = new HashMap<>();
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
        CheckedRunnable<IOException> lookupAndCheck = () -> {
            try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                Map<String, Long> liveOps = latestOps.entrySet().stream().filter(e -> e.getValue().operationType() == Engine.Operation.TYPE.INDEX).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().seqNo()));
                assertThat(getDocIds(engine, true).stream().collect(Collectors.toMap(e -> e.getId(), e -> e.getSeqNo())), equalTo(liveOps));
                for (String id : latestOps.keySet()) {
                    String msg = "latestOps=" + latestOps + " op=" + id;
                    DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(id));
                    if (liveOps.containsKey(id) == false) {
                        assertNull(msg, docIdAndSeqNo);
                    } else {
                        assertNotNull(msg, docIdAndSeqNo);
                        assertThat(msg, docIdAndSeqNo.seqNo, equalTo(latestOps.get(id).seqNo()));
                    }
                }
                String notFoundId = randomValueOtherThanMany(liveOps::containsKey, () -> Long.toString(randomNonNegativeLong()));
                assertNull(VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.getIndexReader(), newUid(notFoundId)));
            }
        };
        for (Engine.Operation op : operations) {
            if (op instanceof Engine.Index) {
                engine.index((Engine.Index) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            } else if (op instanceof Engine.Delete) {
                engine.delete((Engine.Delete) op);
                if (latestOps.containsKey(op.id()) == false || latestOps.get(op.id()).seqNo() < op.seqNo()) {
                    latestOps.put(op.id(), op);
                }
            }
            if (randomInt(100) < 10) {
                engine.refresh("test");
                lookupAndCheck.run();
            }
            if (rarely()) {
                engine.flush(false, true);
                lookupAndCheck.run();
            }
        }
        engine.refresh("test");
        lookupAndCheck.run();
    }
}