Examples with PRIMARY org.opensearch.index.engine.Engine.Operation.Origin.PRIMARY used on opensource projects

Example 1 with PRIMARY

use of org.opensearch.index.engine.Engine.Operation.Origin.PRIMARY in project OpenSearch by opensearch-project.

the class InternalEngineTests method testConcurrentGetAndSetOnPrimary.

public void testConcurrentGetAndSetOnPrimary() throws IOException, InterruptedException {
    Thread[] thread = new Thread[randomIntBetween(3, 5)];
    CountDownLatch startGun = new CountDownLatch(thread.length);
    final int opsPerThread = randomIntBetween(10, 20);
    class OpAndVersion {

        final long version;

        final String removed;

        final String added;

        OpAndVersion(long version, String removed, String added) {
            this.version = version;
            this.removed = removed;
            this.added = added;
        }
    }
    final AtomicInteger idGenerator = new AtomicInteger();
    final Queue<OpAndVersion> history = ConcurrentCollections.newQueue();
    ParsedDocument doc = testParsedDocument("1", null, testDocument(), bytesArray(""), null);
    final Term uidTerm = newUid(doc);
    engine.index(indexForDoc(doc));
    final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
    for (int i = 0; i < thread.length; i++) {
        thread[i] = new Thread(() -> {
            startGun.countDown();
            try {
                startGun.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
            for (int op = 0; op < opsPerThread; op++) {
                try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
                    FieldsVisitor visitor = new FieldsVisitor(true);
                    get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
                    List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
                    String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
                    String added = "v_" + idGenerator.incrementAndGet();
                    values.add(added);
                    Engine.Index index = new Engine.Index(uidTerm, testParsedDocument("1", null, testDocument(), bytesArray(Strings.collectionToCommaDelimitedString(values)), null), UNASSIGNED_SEQ_NO, 2, get.version(), VersionType.INTERNAL, PRIMARY, System.currentTimeMillis(), -1, false, UNASSIGNED_SEQ_NO, 0);
                    Engine.IndexResult indexResult = engine.index(index);
                    if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) {
                        history.add(new OpAndVersion(indexResult.getVersion(), removed, added));
                    }
                } catch (IOException e) {
                    throw new AssertionError(e);
                }
            }
        });
        thread[i].start();
    }
    for (int i = 0; i < thread.length; i++) {
        thread[i].join();
    }
    List<OpAndVersion> sortedHistory = new ArrayList<>(history);
    sortedHistory.sort(Comparator.comparing(o -> o.version));
    Set<String> currentValues = new HashSet<>();
    for (int i = 0; i < sortedHistory.size(); i++) {
        OpAndVersion op = sortedHistory.get(i);
        if (i > 0) {
            assertThat("duplicate version", op.version, not(equalTo(sortedHistory.get(i - 1).version)));
        }
        boolean exists = op.removed == null ? true : currentValues.remove(op.removed);
        assertTrue(op.removed + " should exist", exists);
        exists = currentValues.add(op.added);
        assertTrue(op.added + " should not exist", exists);
    }
    try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
        FieldsVisitor visitor = new FieldsVisitor(true);
        get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
        List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
        assertThat(currentValues, equalTo(new HashSet<>(values)));
    }
}

Example 2 with PRIMARY

use of org.opensearch.index.engine.Engine.Operation.Origin.PRIMARY in project OpenSearch by opensearch-project.

the class InternalEngineTests method testNoOps.

/*
     * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
     * are correctly added to the translog.
     */
public void testNoOps() throws IOException {
    engine.close();
    InternalEngine noOpEngine = null;
    final int maxSeqNo = randomIntBetween(0, 128);
    final int localCheckpoint = randomIntBetween(0, maxSeqNo);
    try {
        final BiFunction<Long, Long, LocalCheckpointTracker> supplier = (ms, lcp) -> new LocalCheckpointTracker(maxSeqNo, localCheckpoint);
        EngineConfig noopEngineConfig = copy(engine.config(), new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy()));
        noOpEngine = new InternalEngine(noopEngineConfig, IndexWriter.MAX_DOCS, supplier) {

            @Override
            protected long doGenerateSeqNoForOperation(Operation operation) {
                throw new UnsupportedOperationException();
            }
        };
        noOpEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get());
        final String reason = "filling gaps";
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 1, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled));
        noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 2, primaryTerm.get(), randomFrom(PRIMARY, REPLICA, PEER_RECOVERY), System.nanoTime(), reason));
        assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled + 1));
        // skip to the op that we added to the translog
        Translog.Operation op;
        Translog.Operation last = null;
        try (Translog.Snapshot snapshot = noOpEngine.getTranslog().newSnapshot()) {
            while ((op = snapshot.next()) != null) {
                last = op;
            }
        }
        assertNotNull(last);
        assertThat(last, instanceOf(Translog.NoOp.class));
        final Translog.NoOp noOp = (Translog.NoOp) last;
        assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
        assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
        assertThat(noOp.reason(), equalTo(reason));
        if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
            MapperService mapperService = createMapperService("test");
            List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
            // fills n gap and 2 manual noop.
            assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint));
            for (int i = 0; i < operationsFromLucene.size(); i++) {
                assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
            }
            assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
        }
    } finally {
        IOUtils.close(noOpEngine);
    }
}