Examples with LocalCheckpointTracker org.opensearch.index.seqno.LocalCheckpointTracker used on opensource projects

Example 1 with LocalCheckpointTracker

use of org.opensearch.index.seqno.LocalCheckpointTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testRebuildLocalCheckpointTrackerAndVersionMap.

public void testRebuildLocalCheckpointTrackerAndVersionMap() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    Path translogPath = createTempDir();
    List<Engine.Operation> operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean());
    List<List<Engine.Operation>> commits = new ArrayList<>();
    commits.add(new ArrayList<>());
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
        final List<DocIdSeqNoAndSource> docs;
        try (InternalEngine engine = createEngine(config)) {
            List<Engine.Operation> flushedOperations = new ArrayList<>();
            for (Engine.Operation op : operations) {
                flushedOperations.add(op);
                applyOperation(engine, op);
                if (randomBoolean()) {
                    engine.syncTranslog();
                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                }
                if (randomInt(100) < 10) {
                    engine.refresh("test");
                }
                if (randomInt(100) < 5) {
                    engine.flush(true, true);
                    flushedOperations.sort(Comparator.comparing(Engine.Operation::seqNo));
                    commits.add(new ArrayList<>(flushedOperations));
                }
            }
            docs = getDocIds(engine, true);
        }
        List<Engine.Operation> operationsInSafeCommit = null;
        for (int i = commits.size() - 1; i >= 0; i--) {
            if (commits.get(i).stream().allMatch(op -> op.seqNo() <= globalCheckpoint.get())) {
                operationsInSafeCommit = commits.get(i);
                break;
            }
        }
        assertThat(operationsInSafeCommit, notNullValue());
        try (InternalEngine engine = new InternalEngine(config)) {
            // do not recover from translog
            final Map<BytesRef, Engine.Operation> deletesAfterCheckpoint = new HashMap<>();
            for (Engine.Operation op : operationsInSafeCommit) {
                if (op instanceof Engine.NoOp == false && op.seqNo() > engine.getPersistedLocalCheckpoint()) {
                    deletesAfterCheckpoint.put(new Term(IdFieldMapper.NAME, Uid.encodeId(op.id())).bytes(), op);
                }
            }
            deletesAfterCheckpoint.values().removeIf(o -> o instanceof Engine.Delete == false);
            final Map<BytesRef, VersionValue> versionMap = engine.getVersionMap();
            for (BytesRef uid : deletesAfterCheckpoint.keySet()) {
                final VersionValue versionValue = versionMap.get(uid);
                final Engine.Operation op = deletesAfterCheckpoint.get(uid);
                final String msg = versionValue + " vs " + "op[" + op.operationType() + "id=" + op.id() + " seqno=" + op.seqNo() + " term=" + op.primaryTerm() + "]";
                assertThat(versionValue, instanceOf(DeleteVersionValue.class));
                assertThat(msg, versionValue.seqNo, equalTo(op.seqNo()));
                assertThat(msg, versionValue.term, equalTo(op.primaryTerm()));
                assertThat(msg, versionValue.version, equalTo(op.version()));
            }
            assertThat(versionMap.keySet(), equalTo(deletesAfterCheckpoint.keySet()));
            final LocalCheckpointTracker tracker = engine.getLocalCheckpointTracker();
            final Set<Long> seqNosInSafeCommit = operationsInSafeCommit.stream().map(op -> op.seqNo()).collect(Collectors.toSet());
            for (Engine.Operation op : operations) {
                assertThat("seq_no=" + op.seqNo() + " max_seq_no=" + tracker.getMaxSeqNo() + "checkpoint=" + tracker.getProcessedCheckpoint(), tracker.hasProcessed(op.seqNo()), equalTo(seqNosInSafeCommit.contains(op.seqNo())));
            }
            engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
            assertThat(getDocIds(engine, true), equalTo(docs));
        }
    }
}

Example 2 with LocalCheckpointTracker

use of org.opensearch.index.seqno.LocalCheckpointTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testTranslogRecoveryWithMultipleGenerations.

public void testTranslogRecoveryWithMultipleGenerations() throws IOException {
    final int docs = randomIntBetween(1, 4096);
    final List<Long> seqNos = LongStream.range(0, docs).boxed().collect(Collectors.toList());
    Randomness.shuffle(seqNos);
    Engine initialEngine = null;
    Engine recoveringEngine = null;
    Store store = createStore();
    final AtomicInteger counter = new AtomicInteger();
    try {
        initialEngine = createEngine(store, createTempDir(), LocalCheckpointTracker::new, (engine, operation) -> seqNos.get(counter.getAndIncrement()));
        for (int i = 0; i < docs; i++) {
            final String id = Integer.toString(i);
            final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
            initialEngine.index(indexForDoc(doc));
            if (rarely()) {
                getTranslog(initialEngine).rollGeneration();
            } else if (rarely()) {
                initialEngine.flush();
            }
        }
        initialEngine.close();
        recoveringEngine = new InternalEngine(initialEngine.config());
        recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        recoveringEngine.refresh("test");
        try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) {
            TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), docs);
            assertEquals(docs, topDocs.totalHits.value);
        }
    } finally {
        IOUtils.close(initialEngine, recoveringEngine, store);
    }
}

Example 3 with LocalCheckpointTracker

use of org.opensearch.index.seqno.LocalCheckpointTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSeqNoGenerator.

public void testSeqNoGenerator() throws IOException {
    engine.close();
    final long seqNo = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Integer.MAX_VALUE);
    final BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier = (ms, lcp) -> new LocalCheckpointTracker(SequenceNumbers.NO_OPS_PERFORMED, SequenceNumbers.NO_OPS_PERFORMED);
    final AtomicLong seqNoGenerator = new AtomicLong(seqNo);
    try (Engine e = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, localCheckpointTrackerSupplier, null, (engine, operation) -> seqNoGenerator.getAndIncrement())) {
        final String id = "id";
        final Field uidField = new Field("_id", id, IdFieldMapper.Defaults.FIELD_TYPE);
        final String type = "type";
        final Field versionField = new NumericDocValuesField("_version", 0);
        final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
        final ParseContext.Document document = new ParseContext.Document();
        document.add(uidField);
        document.add(versionField);
        document.add(seqID.seqNo);
        document.add(seqID.seqNoDocValue);
        document.add(seqID.primaryTerm);
        final BytesReference source = new BytesArray(new byte[] { 1 });
        final ParsedDocument parsedDocument = new ParsedDocument(versionField, seqID, id, type, "routing", Collections.singletonList(document), source, XContentType.JSON, null);
        final Engine.Index index = new Engine.Index(new Term("_id", parsedDocument.id()), parsedDocument, UNASSIGNED_SEQ_NO, randomIntBetween(1, 8), Versions.NOT_FOUND, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, randomBoolean(), UNASSIGNED_SEQ_NO, 0);
        final Engine.IndexResult indexResult = e.index(index);
        assertThat(indexResult.getSeqNo(), equalTo(seqNo));
        assertThat(seqNoGenerator.get(), equalTo(seqNo + 1));
        final Engine.Delete delete = new Engine.Delete(type, id, new Term("_id", parsedDocument.id()), UNASSIGNED_SEQ_NO, randomIntBetween(1, 8), Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0);
        final Engine.DeleteResult deleteResult = e.delete(delete);
        assertThat(deleteResult.getSeqNo(), equalTo(seqNo + 1));
        assertThat(seqNoGenerator.get(), equalTo(seqNo + 2));
    }
}

Example 4 with LocalCheckpointTracker

use of org.opensearch.index.seqno.LocalCheckpointTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testCommitStats.

public void testCommitStats() throws IOException {
    final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final AtomicLong globalCheckpoint = new AtomicLong(UNASSIGNED_SEQ_NO);
    try (Store store = createStore();
        InternalEngine engine = createEngine(store, createTempDir(), (maxSeq, localCP) -> new LocalCheckpointTracker(maxSeq, localCP) {

            @Override
            public long getMaxSeqNo() {
                return maxSeqNo.get();
            }

            @Override
            public long getProcessedCheckpoint() {
                return localCheckpoint.get();
            }
        })) {
        CommitStats stats1 = engine.commitStats();
        assertThat(stats1.getGeneration(), greaterThan(0L));
        assertThat(stats1.getId(), notNullValue());
        assertThat(stats1.getUserData(), hasKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
        assertThat(Long.parseLong(stats1.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
        assertThat(stats1.getUserData(), hasKey(SequenceNumbers.MAX_SEQ_NO));
        assertThat(Long.parseLong(stats1.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
        maxSeqNo.set(rarely() ? SequenceNumbers.NO_OPS_PERFORMED : randomIntBetween(0, 1024));
        localCheckpoint.set(rarely() || maxSeqNo.get() == SequenceNumbers.NO_OPS_PERFORMED ? SequenceNumbers.NO_OPS_PERFORMED : randomIntBetween(0, 1024));
        globalCheckpoint.set(rarely() || localCheckpoint.get() == SequenceNumbers.NO_OPS_PERFORMED ? UNASSIGNED_SEQ_NO : randomIntBetween(0, (int) localCheckpoint.get()));
        engine.flush(true, true);
        CommitStats stats2 = engine.commitStats();
        assertThat(stats2.getGeneration(), greaterThan(stats1.getGeneration()));
        assertThat(stats2.getId(), notNullValue());
        assertThat(stats2.getId(), not(equalTo(stats1.getId())));
        assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY));
        assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY)));
        assertThat(Long.parseLong(stats2.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(localCheckpoint.get()));
        assertThat(stats2.getUserData(), hasKey(SequenceNumbers.MAX_SEQ_NO));
        assertThat(Long.parseLong(stats2.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(maxSeqNo.get()));
    }
}

Example 5 with LocalCheckpointTracker

use of org.opensearch.index.seqno.LocalCheckpointTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testNoOps.

/*
     * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
     * are correctly added to the translog.
     */
public void testNoOps() throws IOException {
    engine.close();
    InternalEngine noOpEngine = null;
    final int maxSeqNo = randomIntBetween(0, 128);
    final int localCheckpoint = randomIntBetween(0, maxSeqNo);
    try {
        final BiFunction<Long, Long, LocalCheckpointTracker> supplier = (ms, lcp) -> new LocalCheckpointTracker(maxSeqNo, localCheckpoint);
        EngineConfig noopEngineConfig = copy(engine.config(), new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy()));
        noOpEngine = new InternalEngine(noopEngineConfig, IndexWriter.MAX_DOCS, supplier) {

            @Override
            protected long doGenerateSeqNoForOperation(Operation operation) {
                throw new UnsupportedOperationException();
            }
        };
        noOpEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get());
        final String reason = "filling gaps";
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 1, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled));
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 2, primaryTerm.get(), randomFrom(PRIMARY, REPLICA, PEER_RECOVERY), System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled + 1));
        // skip to the op that we added to the translog
        Translog.Operation op;
        Translog.Operation last = null;
        try (Translog.Snapshot snapshot = noOpEngine.getTranslog().newSnapshot()) {
            while ((op = snapshot.next()) != null) {
                last = op;
            }
        }
        assertNotNull(last);
        assertThat(last, instanceOf(Translog.NoOp.class));
        final Translog.NoOp noOp = (Translog.NoOp) last;
        assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
        assertThat(noOp.reason(), equalTo(reason));
        if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
            MapperService mapperService = createMapperService("test");
            List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
            // fills n gap and 2 manual noop.
            assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint));
            for (int i = 0; i < operationsFromLucene.size(); i++) {
                assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
            }
            assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
        }
    } finally {
        IOUtils.close(noOpEngine);
    }
}