Examples with REPLICA org.opensearch.index.engine.Engine.Operation.Origin.REPLICA used on opensource projects

Example 1 with REPLICA

use of org.opensearch.index.engine.Engine.Operation.Origin.REPLICA in project OpenSearch by opensearch-project.

the class InternalEngineTests method testTrimUnsafeCommits.

public void testTrimUnsafeCommits() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final int maxSeqNo = 40;
    final List<Long> seqNos = LongStream.rangeClosed(0, maxSeqNo).boxed().collect(Collectors.toList());
    Collections.shuffle(seqNos, random());
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        final List<Long> commitMaxSeqNo = new ArrayList<>();
        final long minTranslogGen;
        try (InternalEngine engine = createEngine(config)) {
            for (int i = 0; i < seqNos.size(); i++) {
                ParsedDocument doc = testParsedDocument(Long.toString(seqNos.get(i)), null, testDocument(), new BytesArray("{}"), null);
                Engine.Index index = new Engine.Index(newUid(doc), doc, seqNos.get(i), 0, 1, null, REPLICA, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
                engine.index(index);
                if (randomBoolean()) {
                    engine.flush();
                    final Long maxSeqNoInCommit = seqNos.subList(0, i + 1).stream().max(Long::compareTo).orElse(-1L);
                    commitMaxSeqNo.add(maxSeqNoInCommit);
                }
            }
            globalCheckpoint.set(randomInt(maxSeqNo));
            engine.syncTranslog();
            minTranslogGen = engine.getTranslog().getMinFileGeneration();
        }
        store.trimUnsafeCommits(globalCheckpoint.get(), minTranslogGen, config.getIndexSettings().getIndexVersionCreated());
        long safeMaxSeqNo = commitMaxSeqNo.stream().filter(s -> s <= globalCheckpoint.get()).reduce(// get the last one.
        (s1, s2) -> s2).orElse(SequenceNumbers.NO_OPS_PERFORMED);
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        assertThat(commits, hasSize(1));
        assertThat(commits.get(0).getUserData().get(SequenceNumbers.MAX_SEQ_NO), equalTo(Long.toString(safeMaxSeqNo)));
        try (IndexReader reader = DirectoryReader.open(commits.get(0))) {
            for (LeafReaderContext context : reader.leaves()) {
                final NumericDocValues values = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
                if (values != null) {
                    for (int docID = 0; docID < context.reader().maxDoc(); docID++) {
                        if (values.advanceExact(docID) == false) {
                            throw new AssertionError("Document does not have a seq number: " + docID);
                        }
                        assertThat(values.longValue(), lessThanOrEqualTo(globalCheckpoint.get()));
                    }
                }
            }
        }
    }
}

Example 2 with REPLICA

use of org.opensearch.index.engine.Engine.Operation.Origin.REPLICA in project OpenSearch by opensearch-project.

the class InternalEngineTests method testTreatDocumentFailureAsFatalError.

public void testTreatDocumentFailureAsFatalError() throws Exception {
    AtomicReference<IOException> addDocException = new AtomicReference<>();
    IndexWriterFactory indexWriterFactory = (dir, iwc) -> new IndexWriter(dir, iwc) {

        @Override
        public long addDocument(Iterable<? extends IndexableField> doc) throws IOException {
            final IOException ex = addDocException.getAndSet(null);
            if (ex != null) {
                throw ex;
            }
            return super.addDocument(doc);
        }
    };
    try (Store store = createStore();
        InternalEngine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, indexWriterFactory)) {
        final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), SOURCE, null);
        Engine.Operation.Origin origin = randomFrom(REPLICA, LOCAL_RESET, PEER_RECOVERY);
        Engine.Index index = new Engine.Index(newUid(doc), doc, randomNonNegativeLong(), primaryTerm.get(), randomNonNegativeLong(), null, origin, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM);
        addDocException.set(new IOException("simulated"));
        expectThrows(IOException.class, () -> engine.index(index));
        assertTrue(engine.isClosed.get());
        assertNotNull(engine.failedEngine.get());
    }
}

Example 3 with REPLICA

use of org.opensearch.index.engine.Engine.Operation.Origin.REPLICA in project OpenSearch by opensearch-project.

the class InternalEngineTests method testNoOps.

/*
     * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
     * are correctly added to the translog.
     */
public void testNoOps() throws IOException {
    engine.close();
    InternalEngine noOpEngine = null;
    final int maxSeqNo = randomIntBetween(0, 128);
    final int localCheckpoint = randomIntBetween(0, maxSeqNo);
    try {
        final BiFunction<Long, Long, LocalCheckpointTracker> supplier = (ms, lcp) -> new LocalCheckpointTracker(maxSeqNo, localCheckpoint);
        EngineConfig noopEngineConfig = copy(engine.config(), new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy()));
        noOpEngine = new InternalEngine(noopEngineConfig, IndexWriter.MAX_DOCS, supplier) {

            @Override
            protected long doGenerateSeqNoForOperation(Operation operation) {
                throw new UnsupportedOperationException();
            }
        };
        noOpEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get());
        final String reason = "filling gaps";
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 1, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled));
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 2, primaryTerm.get(), randomFrom(PRIMARY, REPLICA, PEER_RECOVERY), System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled + 1));
        // skip to the op that we added to the translog
        Translog.Operation op;
        Translog.Operation last = null;
        try (Translog.Snapshot snapshot = noOpEngine.getTranslog().newSnapshot()) {
            while ((op = snapshot.next()) != null) {
                last = op;
            }
        }
        assertNotNull(last);
        assertThat(last, instanceOf(Translog.NoOp.class));
        final Translog.NoOp noOp = (Translog.NoOp) last;
        assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
        assertThat(noOp.reason(), equalTo(reason));
        if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
            MapperService mapperService = createMapperService("test");
            List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
            // fills n gap and 2 manual noop.
            assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint));
            for (int i = 0; i < operationsFromLucene.size(); i++) {
                assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
            }
            assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
        }
    } finally {
        IOUtils.close(noOpEngine);
    }
}