Examples with SeqNoStats org.opensearch.index.seqno.SeqNoStats used on opensource projects

Example 6 with SeqNoStats

use of org.opensearch.index.seqno.SeqNoStats in project OpenSearch by opensearch-project.

the class IndexingIT method assertSeqNoOnShards.

private void assertSeqNoOnShards(String index, Nodes nodes, int numDocs, RestClient client) throws Exception {
    assertBusy(() -> {
        try {
            List<Shard> shards = buildShards(index, nodes, client);
            Shard primaryShard = shards.stream().filter(Shard::isPrimary).findFirst().get();
            assertNotNull("failed to find primary shard", primaryShard);
            final long expectedGlobalCkp = numDocs - 1;
            final long expectMaxSeqNo = numDocs - 1;
            logger.info("primary resolved to node {}", primaryShard.getNode());
            for (Shard shard : shards) {
                final SeqNoStats seqNoStats = shard.getSeqNoStats();
                logger.info("stats for {}, primary [{}]: [{}]", shard.getNode(), shard.isPrimary(), seqNoStats);
                assertThat("max_seq no on " + shard.getNode() + " is wrong", seqNoStats.getMaxSeqNo(), equalTo(expectMaxSeqNo));
                assertThat("localCheckpoint no on " + shard.getNode() + " is wrong", seqNoStats.getLocalCheckpoint(), equalTo(expectMaxSeqNo));
                assertThat("globalCheckpoint no on " + shard.getNode() + " is wrong", seqNoStats.getGlobalCheckpoint(), equalTo(expectedGlobalCkp));
            }
        } catch (IOException e) {
            throw new AssertionError("unexpected io exception", e);
        }
    });
}

Example 7 with SeqNoStats

use of org.opensearch.index.seqno.SeqNoStats in project OpenSearch by opensearch-project.

the class InternalTestCluster method assertSeqNos.

public void assertSeqNos() throws Exception {
    assertBusy(() -> {
        final ClusterState state = clusterService().state();
        for (ObjectObjectCursor<String, IndexRoutingTable> indexRoutingTable : state.routingTable().indicesRouting()) {
            for (IntObjectCursor<IndexShardRoutingTable> indexShardRoutingTable : indexRoutingTable.value.shards()) {
                ShardRouting primaryShardRouting = indexShardRoutingTable.value.primaryShard();
                final IndexShard primaryShard = getShardOrNull(state, primaryShardRouting);
                if (primaryShard == null) {
                    // just ignore - shard movement
                    continue;
                }
                final SeqNoStats primarySeqNoStats;
                final ObjectLongMap<String> syncGlobalCheckpoints;
                try {
                    primarySeqNoStats = primaryShard.seqNoStats();
                    syncGlobalCheckpoints = primaryShard.getInSyncGlobalCheckpoints();
                } catch (AlreadyClosedException ex) {
                    // shard is closed - just ignore
                    continue;
                }
                assertThat(primaryShardRouting + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)));
                for (ShardRouting replicaShardRouting : indexShardRoutingTable.value.replicaShards()) {
                    final IndexShard replicaShard = getShardOrNull(state, replicaShardRouting);
                    if (replicaShard == null) {
                        // just ignore - shard movement
                        continue;
                    }
                    final SeqNoStats seqNoStats;
                    try {
                        seqNoStats = replicaShard.seqNoStats();
                    } catch (AlreadyClosedException e) {
                        // shard is closed - just ignore
                        continue;
                    }
                    assertThat(replicaShardRouting + " seq_no_stats mismatch", seqNoStats, equalTo(primarySeqNoStats));
                    // the local knowledge on the primary of the global checkpoint equals the global checkpoint on the shard
                    assertThat(replicaShardRouting + " global checkpoint syncs mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(syncGlobalCheckpoints.get(replicaShardRouting.allocationId().getId())));
                }
            }
        }
    }, 30, TimeUnit.SECONDS);
}

Example 8 with SeqNoStats

use of org.opensearch.index.seqno.SeqNoStats in project OpenSearch by opensearch-project.

the class InternalEngineTests method testRestoreLocalHistoryFromTranslog.

public void testRestoreLocalHistoryFromTranslog() throws IOException {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        final ArrayList<Long> seqNos = new ArrayList<>();
        final int numOps = randomIntBetween(0, 1024);
        for (int i = 0; i < numOps; i++) {
            if (rarely()) {
                continue;
            }
            seqNos.add((long) i);
        }
        Randomness.shuffle(seqNos);
        final EngineConfig engineConfig;
        final SeqNoStats prevSeqNoStats;
        final List<DocIdSeqNoAndSource> prevDocs;
        try (InternalEngine engine = createEngine(store, createTempDir(), globalCheckpoint::get)) {
            engineConfig = engine.config();
            for (final long seqNo : seqNos) {
                final String id = Long.toString(seqNo);
                final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                engine.index(replicaIndexForDoc(doc, 1, seqNo, false));
                if (rarely()) {
                    engine.rollTranslogGeneration();
                }
                if (rarely()) {
                    engine.flush();
                }
            }
            globalCheckpoint.set(randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, engine.getPersistedLocalCheckpoint()));
            engine.syncTranslog();
            prevSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            prevDocs = getDocIds(engine, true);
        }
        try (InternalEngine engine = new InternalEngine(engineConfig)) {
            final long currentTranslogGeneration = engine.getTranslog().currentFileGeneration();
            engine.recoverFromTranslog(translogHandler, globalCheckpoint.get());
            engine.restoreLocalHistoryFromTranslog(translogHandler);
            assertThat(getDocIds(engine, true), equalTo(prevDocs));
            SeqNoStats seqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            assertThat(seqNoStats.getLocalCheckpoint(), equalTo(prevSeqNoStats.getLocalCheckpoint()));
            assertThat(seqNoStats.getMaxSeqNo(), equalTo(prevSeqNoStats.getMaxSeqNo()));
            assertThat("restore from local translog must not add operations to translog", engine.getTranslog().totalOperationsByMinGen(currentTranslogGeneration), equalTo(0));
        }
        assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, createMapperService("test"));
    }
}

Example 9 with SeqNoStats

use of org.opensearch.index.seqno.SeqNoStats in project OpenSearch by opensearch-project.

the class ReadOnlyEngineTests method testReadOnlyEngine.

public void testReadOnlyEngine() throws Exception {
    IOUtils.close(engine, store);
    Engine readOnlyEngine = null;
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        int numDocs = scaledRandomIntBetween(10, 1000);
        final SeqNoStats lastSeqNoStats;
        final List<DocIdSeqNoAndSource> lastDocIds;
        try (InternalEngine engine = createEngine(config)) {
            Engine.Get get = null;
            for (int i = 0; i < numDocs; i++) {
                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                if (get == null || rarely()) {
                    get = newGet(randomBoolean(), doc);
                }
                if (rarely()) {
                    engine.flush();
                }
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            }
            engine.syncTranslog();
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            engine.flush();
            readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, engine.getSeqNoStats(globalCheckpoint.get()), engine.getTranslogStats(), false, Function.identity(), true);
            lastSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            lastDocIds = getDocIds(engine, true);
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
            for (int i = 0; i < numDocs; i++) {
                if (randomBoolean()) {
                    String delId = Integer.toString(i);
                    engine.delete(new Engine.Delete("test", delId, newUid(delId), primaryTerm.get()));
                }
                if (rarely()) {
                    engine.flush();
                }
            }
            Engine.Searcher external = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL);
            Engine.Searcher internal = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
            assertSame(external.getIndexReader(), internal.getIndexReader());
            assertThat(external.getIndexReader(), instanceOf(DirectoryReader.class));
            DirectoryReader dirReader = external.getDirectoryReader();
            OpenSearchDirectoryReader esReader = getOpenSearchDirectoryReader(dirReader);
            IndexReader.CacheHelper helper = esReader.getReaderCacheHelper();
            assertNotNull(helper);
            assertEquals(helper.getKey(), dirReader.getReaderCacheHelper().getKey());
            IOUtils.close(external, internal);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
            try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
                assertTrue(getResult.exists());
            }
        }
        // Close and reopen the main engine
        try (InternalEngine recoveringEngine = new InternalEngine(config)) {
            recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
        }
    } finally {
        IOUtils.close(readOnlyEngine);
    }
}

Example 10 with SeqNoStats

use of org.opensearch.index.seqno.SeqNoStats in project OpenSearch by opensearch-project.

the class IndexShardTests method testOnCloseStats.

public void testOnCloseStats() throws IOException {
    final IndexShard indexShard = newStartedShard(true);
    for (int i = 0; i < 3; i++) {
        indexDoc(indexShard, "_doc", "" + i, "{\"foo\" : \"" + randomAlphaOfLength(10) + "\"}");
        // produce segments
        indexShard.refresh("test");
    }
    // check stats on closed and on opened shard
    if (randomBoolean()) {
        closeShards(indexShard);
        expectThrows(AlreadyClosedException.class, () -> indexShard.seqNoStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.commitStats());
        expectThrows(AlreadyClosedException.class, () -> indexShard.storeStats());
    } else {
        final SeqNoStats seqNoStats = indexShard.seqNoStats();
        assertThat(seqNoStats.getLocalCheckpoint(), equalTo(2L));
        final CommitStats commitStats = indexShard.commitStats();
        assertThat(commitStats.getGeneration(), equalTo(2L));
        final StoreStats storeStats = indexShard.storeStats();
        assertThat(storeStats.sizeInBytes(), greaterThan(0L));
        closeShards(indexShard);
    }
}