Examples with SeqNoStats org.elasticsearch.index.seqno.SeqNoStats used on opensource projects

Example 1 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project elasticsearch by elastic.

the class PeerRecoveryTargetService method getStartingSeqNo.

/**
     * Get the starting sequence number for a sequence-number-based request.
     *
     * @param recoveryTarget the target of the recovery
     * @return the starting sequence number or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO} if obtaining the starting sequence number
     * failed
     */
public static long getStartingSeqNo(final RecoveryTarget recoveryTarget) {
    try {
        final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.indexShard().shardPath().resolveTranslog());
        final SeqNoStats seqNoStats = recoveryTarget.store().loadSeqNoStats(globalCheckpoint);
        if (seqNoStats.getMaxSeqNo() <= seqNoStats.getGlobalCheckpoint()) {
            // Recovery will start at the first op after the local check point stored in the commit.
            return seqNoStats.getLocalCheckpoint() + 1;
        } else {
            return SequenceNumbersService.UNASSIGNED_SEQ_NO;
        }
    } catch (final IOException e) {
        // proceeds to attempt a sequence-number-based recovery
        return SequenceNumbersService.UNASSIGNED_SEQ_NO;
    }
}

Example 2 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project elasticsearch by elastic.

the class IndexingIT method assertSeqNoOnShards.

private void assertSeqNoOnShards(Nodes nodes, boolean checkGlobalCheckpoints, int numDocs, RestClient client) throws Exception {
    assertBusy(() -> {
        try {
            List<Shard> shards = buildShards(nodes, client);
            Shard primaryShard = shards.stream().filter(Shard::isPrimary).findFirst().get();
            assertNotNull("failed to find primary shard", primaryShard);
            final long expectedGlobalCkp;
            final long expectMaxSeqNo;
            logger.info("primary resolved to node {}", primaryShard.getNode());
            if (primaryShard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
                expectMaxSeqNo = numDocs - 1;
                expectedGlobalCkp = numDocs - 1;
            } else {
                expectedGlobalCkp = SequenceNumbersService.UNASSIGNED_SEQ_NO;
                expectMaxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
            }
            for (Shard shard : shards) {
                if (shard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
                    final SeqNoStats seqNoStats = shard.getSeqNoStats();
                    logger.info("stats for {}, primary [{}]: [{}]", shard.getNode(), shard.isPrimary(), seqNoStats);
                    assertThat("max_seq no on " + shard.getNode() + " is wrong", seqNoStats.getMaxSeqNo(), equalTo(expectMaxSeqNo));
                    assertThat("localCheckpoint no on " + shard.getNode() + " is wrong", seqNoStats.getLocalCheckpoint(), equalTo(expectMaxSeqNo));
                    if (checkGlobalCheckpoints) {
                        assertThat("globalCheckpoint no on " + shard.getNode() + " is wrong", seqNoStats.getGlobalCheckpoint(), equalTo(expectedGlobalCkp));
                    }
                } else {
                    logger.info("skipping seq no test on {}", shard.getNode());
                }
            }
        } catch (IOException e) {
            throw new AssertionError("unexpected io exception", e);
        }
    });
}

Example 3 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project elasticsearch by elastic.

the class IndexLevelReplicationTests method testCheckpointsAdvance.

public void testCheckpointsAdvance() throws Exception {
    try (ReplicationGroup shards = createGroup(randomInt(3))) {
        shards.startPrimary();
        int numDocs = 0;
        int startedShards;
        do {
            numDocs += shards.indexDocs(randomInt(20));
            startedShards = shards.startReplicas(randomIntBetween(1, 2));
        } while (startedShards > 0);
        if (numDocs == 0 || randomBoolean()) {
            // in the case we have no indexing, we simulate the background global checkpoint sync
            shards.getPrimary().updateGlobalCheckpointOnPrimary();
        }
        for (IndexShard shard : shards) {
            final SeqNoStats shardStats = shard.seqNoStats();
            final ShardRouting shardRouting = shard.routingEntry();
            logger.debug("seq_no stats for {}: {}", shardRouting, XContentHelper.toString(shardStats, new ToXContent.MapParams(Collections.singletonMap("pretty", "false"))));
            assertThat(shardRouting + " local checkpoint mismatch", shardStats.getLocalCheckpoint(), equalTo(numDocs - 1L));
            assertThat(shardRouting + " global checkpoint mismatch", shardStats.getGlobalCheckpoint(), equalTo(numDocs - 1L));
            assertThat(shardRouting + " max seq no mismatch", shardStats.getMaxSeqNo(), equalTo(numDocs - 1L));
        }
    }
}

Example 4 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project crate by crate.

the class InternalTestCluster method assertSeqNos.

public void assertSeqNos() throws Exception {
    assertBusy(() -> {
        final ClusterState state = clusterService().state();
        for (ObjectObjectCursor<String, IndexRoutingTable> indexRoutingTable : state.routingTable().indicesRouting()) {
            for (IntObjectCursor<IndexShardRoutingTable> indexShardRoutingTable : indexRoutingTable.value.shards()) {
                ShardRouting primaryShardRouting = indexShardRoutingTable.value.primaryShard();
                final IndexShard primaryShard = getShardOrNull(state, primaryShardRouting);
                if (primaryShard == null) {
                    // just ignore - shard movement
                    continue;
                }
                final SeqNoStats primarySeqNoStats;
                final ObjectLongMap<String> syncGlobalCheckpoints;
                try {
                    primarySeqNoStats = primaryShard.seqNoStats();
                    syncGlobalCheckpoints = primaryShard.getInSyncGlobalCheckpoints();
                } catch (AlreadyClosedException ex) {
                    // shard is closed - just ignore
                    continue;
                }
                assertThat(primaryShardRouting + " should have set the global checkpoint", primarySeqNoStats.getGlobalCheckpoint(), not(equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)));
                for (ShardRouting replicaShardRouting : indexShardRoutingTable.value.replicaShards()) {
                    final IndexShard replicaShard = getShardOrNull(state, replicaShardRouting);
                    if (replicaShard == null) {
                        // just ignore - shard movement
                        continue;
                    }
                    final SeqNoStats seqNoStats;
                    try {
                        seqNoStats = replicaShard.seqNoStats();
                    } catch (AlreadyClosedException e) {
                        // shard is closed - just ignore
                        continue;
                    }
                    assertThat(replicaShardRouting + " seq_no_stats mismatch", seqNoStats, equalTo(primarySeqNoStats));
                    // the local knowledge on the primary of the global checkpoint equals the global checkpoint on the shard
                    assertThat(replicaShardRouting + " global checkpoint syncs mismatch", seqNoStats.getGlobalCheckpoint(), equalTo(syncGlobalCheckpoints.get(replicaShardRouting.allocationId().getId())));
                }
            }
        }
    }, 60, TimeUnit.SECONDS);
}

Example 5 with SeqNoStats

use of org.elasticsearch.index.seqno.SeqNoStats in project crate by crate.

the class InternalEngineTests method testRestoreLocalHistoryFromTranslog.

@Test
public void testRestoreLocalHistoryFromTranslog() throws IOException {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        final ArrayList<Long> seqNos = new ArrayList<>();
        final int numOps = randomIntBetween(0, 1024);
        for (int i = 0; i < numOps; i++) {
            if (rarely()) {
                continue;
            }
            seqNos.add((long) i);
        }
        Randomness.shuffle(seqNos);
        final EngineConfig engineConfig;
        final SeqNoStats prevSeqNoStats;
        final List<DocIdSeqNoAndSource> prevDocs;
        try (InternalEngine engine = createEngine(store, createTempDir(), globalCheckpoint::get)) {
            engineConfig = engine.config();
            for (final long seqNo : seqNos) {
                final String id = Long.toString(seqNo);
                final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                engine.index(replicaIndexForDoc(doc, 1, seqNo, false));
                if (rarely()) {
                    engine.rollTranslogGeneration();
                }
                if (rarely()) {
                    engine.flush();
                }
            }
            globalCheckpoint.set(randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, engine.getPersistedLocalCheckpoint()));
            engine.syncTranslog();
            prevSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            prevDocs = getDocIds(engine, true);
        }
        try (InternalEngine engine = new InternalEngine(engineConfig)) {
            final long currentTranslogGeneration = engine.getTranslog().currentFileGeneration();
            engine.recoverFromTranslog(translogHandler, globalCheckpoint.get());
            engine.restoreLocalHistoryFromTranslog(translogHandler);
            assertThat(getDocIds(engine, true), equalTo(prevDocs));
            SeqNoStats seqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            assertThat(seqNoStats.getLocalCheckpoint(), equalTo(prevSeqNoStats.getLocalCheckpoint()));
            assertThat(seqNoStats.getMaxSeqNo(), equalTo(prevSeqNoStats.getMaxSeqNo()));
            assertThat("restore from local translog must not add operations to translog", engine.getTranslog().totalOperationsByMinGen(currentTranslogGeneration), equalTo(0));
        }
        assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, createMapperService("test"));
    }
}