Examples with SeqNoStats org.elasticsearch.index.seqno.SeqNoStats used on opensource projects

Example 6 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project crate by crate.

the class PeerRecoveryTargetServiceTests method testResetStartRequestIfTranslogIsCorrupted.

@Test
public void testResetStartRequestIfTranslogIsCorrupted() throws Exception {
    DiscoveryNode pNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
    DiscoveryNode rNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
    IndexShard shard = newStartedShard(false);
    final SeqNoStats seqNoStats = populateRandomData(shard);
    shard.close("test", false);
    if (randomBoolean()) {
        shard.store().associateIndexWithNewTranslog(UUIDs.randomBase64UUID());
    } else if (randomBoolean()) {
        Translog.createEmptyTranslog(shard.shardPath().resolveTranslog(), seqNoStats.getGlobalCheckpoint(), shard.shardId(), shard.getOperationPrimaryTerm());
    } else {
        IOUtils.rm(shard.shardPath().resolveTranslog());
    }
    shard = reinitShard(shard, ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.PeerRecoverySource.INSTANCE));
    shard.markAsRecovering("peer recovery", new RecoveryState(shard.routingEntry(), pNode, rNode));
    shard.prepareForIndexRecovery();
    RecoveryTarget recoveryTarget = new RecoveryTarget(shard, null, null);
    StartRecoveryRequest request = PeerRecoveryTargetService.getStartRecoveryRequest(logger, rNode, recoveryTarget, randomNonNegativeLong());
    assertThat(request.startingSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
    assertThat(request.metadataSnapshot(), sameInstance(Store.MetadataSnapshot.EMPTY));
    recoveryTarget.decRef();
    closeShards(shard);
}

Example 7 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project crate by crate.

the class PeerRecoveryTargetServiceTests method test_prepare_old_index_for_peer_recovery_without_safe_commit.

@Test
public void test_prepare_old_index_for_peer_recovery_without_safe_commit() throws Exception {
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
    var shard = newStartedShard(false, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_3_0_1).build());
    SeqNoStats seqNoStats = populateRandomData(shard);
    var globalCheckpoint = UNASSIGNED_SEQ_NO;
    var replica = reinitShard(shard, ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.PeerRecoverySource.INSTANCE));
    String translogUUID = Translog.createEmptyTranslog(replica.shardPath().resolveTranslog(), globalCheckpoint, replica.shardId(), replica.getPendingPrimaryTerm());
    replica.store().associateIndexWithNewTranslog(translogUUID);
    var safeCommit = replica.store().findSafeIndexCommit(globalCheckpoint);
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    replica.prepareForIndexRecovery();
    assertThat(safeCommit.isPresent(), is(false));
    assertThat(replica.recoverLocallyUpToGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
    assertThat(replica.recoveryState().getTranslog().totalLocal(), equalTo(RecoveryState.Translog.UNKNOWN));
    assertThat(replica.recoveryState().getTranslog().recoveredOperations(), equalTo(0));
    assertThat(replica.getLastKnownGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
    assertThat(replica.recoveryState().getStage(), equalTo(RecoveryState.Stage.TRANSLOG));
    closeShards(replica);
}

Example 8 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project crate by crate.

the class ReadOnlyEngineTests method testReadOnlyEngine.

@Test
public void testReadOnlyEngine() throws Exception {
    IOUtils.close(engine, store);
    Engine readOnlyEngine = null;
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        int numDocs = scaledRandomIntBetween(10, 1000);
        final SeqNoStats lastSeqNoStats;
        final List<DocIdSeqNoAndSource> lastDocIds;
        try (InternalEngine engine = createEngine(config)) {
            Engine.Get get = null;
            for (int i = 0; i < numDocs; i++) {
                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                if (get == null || rarely()) {
                    get = newGet(doc);
                }
                if (rarely()) {
                    engine.flush();
                }
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            }
            engine.syncTranslog();
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            engine.flush();
            readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, engine.getSeqNoStats(globalCheckpoint.get()), engine.getTranslogStats(), false, Function.identity());
            lastSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            lastDocIds = getDocIds(engine, true);
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
            for (int i = 0; i < numDocs; i++) {
                if (randomBoolean()) {
                    String delId = Integer.toString(i);
                    engine.delete(new Engine.Delete(delId, newUid(delId), primaryTerm.get()));
                }
                if (rarely()) {
                    engine.flush();
                }
            }
            Engine.Searcher external = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL);
            Engine.Searcher internal = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
            assertSame(external.getIndexReader(), internal.getIndexReader());
            assertThat(external.getIndexReader(), instanceOf(DirectoryReader.class));
            DirectoryReader dirReader = external.getDirectoryReader();
            ElasticsearchDirectoryReader esReader = getElasticsearchDirectoryReader(dirReader);
            IndexReader.CacheHelper helper = esReader.getReaderCacheHelper();
            assertNotNull(helper);
            assertEquals(helper.getKey(), dirReader.getReaderCacheHelper().getKey());
            IOUtils.close(external, internal);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
        }
        // Close and reopen the main engine
        try (InternalEngine recoveringEngine = new InternalEngine(config)) {
            recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
        }
    } finally {
        IOUtils.close(readOnlyEngine);
    }
}

Example 9 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project elasticsearch by elastic.

the class RelocationIT method beforeIndexDeletion.

@Override
protected void beforeIndexDeletion() throws Exception {
    super.beforeIndexDeletion();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().clear().get();
        for (IndexStats indexStats : stats.getIndices().values()) {
            for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) {
                Optional<ShardStats> maybePrimary = Stream.of(indexShardStats.getShards()).filter(s -> s.getShardRouting().active() && s.getShardRouting().primary()).findFirst();
                if (maybePrimary.isPresent() == false) {
                    continue;
                }
                ShardStats primary = maybePrimary.get();
                final SeqNoStats primarySeqNoStats = primary.getSeqNoStats();
                assertThat(primary.getShardRouting() + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO)));
                for (ShardStats shardStats : indexShardStats) {
                    final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
                    assertThat(shardStats.getShardRouting() + " local checkpoint mismatch", seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " global checkpoint mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                    assertThat(shardStats.getShardRouting() + " max seq no mismatch", seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
                }
            }
        }
    });
}

Example 10 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project elasticsearch by elastic.

the class IndexingIT method buildShards.

private List<Shard> buildShards(Nodes nodes, RestClient client) throws IOException {
    Response response = client.performRequest("GET", "test/_stats", singletonMap("level", "shards"));
    List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices.test.shards.0");
    ArrayList<Shard> shards = new ArrayList<>();
    for (Object shard : shardStats) {
        final String nodeId = ObjectPath.evaluate(shard, "routing.node");
        final Boolean primary = ObjectPath.evaluate(shard, "routing.primary");
        final Node node = nodes.getSafe(nodeId);
        final SeqNoStats seqNoStats;
        if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
            Integer maxSeqNo = ObjectPath.evaluate(shard, "seq_no.max_seq_no");
            Integer localCheckpoint = ObjectPath.evaluate(shard, "seq_no.local_checkpoint");
            Integer globalCheckpoint = ObjectPath.evaluate(shard, "seq_no.global_checkpoint");
            seqNoStats = new SeqNoStats(maxSeqNo, localCheckpoint, globalCheckpoint);
        } else {
            seqNoStats = null;
        }
        shards.add(new Shard(node, primary, seqNoStats));
    }
    return shards;
}