Examples with GatedCloseable org.opensearch.common.concurrent.GatedCloseable used on opensource projects

Example 1 with GatedCloseable

use of org.opensearch.common.concurrent.GatedCloseable in project OpenSearch by opensearch-project.

the class IndexShard method resetEngineToGlobalCheckpoint.

/**
 * Rollback the current engine to the safe commit, then replay local translog up to the global checkpoint.
 */
void resetEngineToGlobalCheckpoint() throws IOException {
    assert Thread.holdsLock(mutex) == false : "resetting engine under mutex";
    assert getActiveOperationsCount() == OPERATIONS_BLOCKED : "resetting engine without blocking operations; active operations are [" + getActiveOperations() + ']';
    // persist the global checkpoint to disk
    sync();
    final SeqNoStats seqNoStats = seqNoStats();
    final TranslogStats translogStats = translogStats();
    // flush to make sure the latest commit, which will be opened by the read-only engine, includes all operations.
    flush(new FlushRequest().waitIfOngoing(true));
    SetOnce<Engine> newEngineReference = new SetOnce<>();
    final long globalCheckpoint = getLastKnownGlobalCheckpoint();
    assert globalCheckpoint == getLastSyncedGlobalCheckpoint();
    synchronized (engineMutex) {
        verifyNotClosed();
        // we must create both new read-only engine and new read-write engine under engineMutex to ensure snapshotStoreMetadata,
        // acquireXXXCommit and close works.
        final Engine readOnlyEngine = new ReadOnlyEngine(newEngineConfig(replicationTracker), seqNoStats, translogStats, false, Function.identity(), true) {

            @Override
            public GatedCloseable<IndexCommit> acquireLastIndexCommit(boolean flushFirst) {
                synchronized (engineMutex) {
                    if (newEngineReference.get() == null) {
                        throw new AlreadyClosedException("engine was closed");
                    }
                    // ignore flushFirst since we flushed above and we do not want to interfere with ongoing translog replay
                    return newEngineReference.get().acquireLastIndexCommit(false);
                }
            }

            @Override
            public GatedCloseable<IndexCommit> acquireSafeIndexCommit() {
                synchronized (engineMutex) {
                    if (newEngineReference.get() == null) {
                        throw new AlreadyClosedException("engine was closed");
                    }
                    return newEngineReference.get().acquireSafeIndexCommit();
                }
            }

            @Override
            public void close() throws IOException {
                assert Thread.holdsLock(engineMutex);
                Engine newEngine = newEngineReference.get();
                if (newEngine == currentEngineReference.get()) {
                    // we successfully installed the new engine so do not close it.
                    newEngine = null;
                }
                IOUtils.close(super::close, newEngine);
            }
        };
        IOUtils.close(currentEngineReference.getAndSet(readOnlyEngine));
        newEngineReference.set(engineFactory.newReadWriteEngine(newEngineConfig(replicationTracker)));
        onNewEngine(newEngineReference.get());
    }
    final Engine.TranslogRecoveryRunner translogRunner = (engine, snapshot) -> runTranslogRecovery(engine, snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
    // TODO: add a dedicate recovery stats for the reset translog
    });
    newEngineReference.get().recoverFromTranslog(translogRunner, globalCheckpoint);
    newEngineReference.get().refresh("reset_engine");
    synchronized (engineMutex) {
        verifyNotClosed();
        IOUtils.close(currentEngineReference.getAndSet(newEngineReference.get()));
        // We set active because we are now writing operations to the engine; this way,
        // if we go idle after some time and become inactive, we still give sync'd flush a chance to run.
        active.set(true);
    }
    // time elapses after the engine is created above (pulling the config settings) until we set the engine reference, during
    // which settings changes could possibly have happened, so here we forcefully push any config changes to the new engine.
    onSettingsChanged();
}

Example 2 with GatedCloseable

use of org.opensearch.common.concurrent.GatedCloseable in project OpenSearch by opensearch-project.

the class RecoverySourceHandler method recoverToTarget.

/**
 * performs the recovery from the local engine to the target
 */
public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
    addListener(listener);
    final Closeable releaseResources = () -> IOUtils.close(resources);
    try {
        cancellableThreads.setOnCancel((reason, beforeCancelEx) -> {
            final RuntimeException e;
            if (shard.state() == IndexShardState.CLOSED) {
                // check if the shard got closed on us
                e = new IndexShardClosedException(shard.shardId(), "shard is closed and recovery was canceled reason [" + reason + "]");
            } else {
                e = new CancellableThreads.ExecutionCancelledException("recovery was canceled reason [" + reason + "]");
            }
            if (beforeCancelEx != null) {
                e.addSuppressed(beforeCancelEx);
            }
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
            throw e;
        });
        final Consumer<Exception> onFailure = e -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[onFailure]");
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
        };
        final SetOnce<RetentionLease> retentionLeaseRef = new SetOnce<>();
        runUnderPrimaryPermit(() -> {
            final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable();
            ShardRouting targetShardRouting = routingTable.getByAllocationId(request.targetAllocationId());
            if (targetShardRouting == null) {
                logger.debug("delaying recovery of {} as it is not listed as assigned to target node {}", request.shardId(), request.targetNode());
                throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node");
            }
            assert targetShardRouting.initializing() : "expected recovery target to be initializing but was " + targetShardRouting;
            retentionLeaseRef.set(shard.getRetentionLeases().get(ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting)));
        }, shardId + " validating recovery target [" + request.targetAllocationId() + "] registered ", shard, cancellableThreads, logger);
        final Closeable retentionLock = shard.acquireHistoryRetentionLock();
        resources.add(retentionLock);
        final long startingSeqNo;
        final boolean isSequenceNumberBasedRecovery = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && isTargetSameHistory() && shard.hasCompleteHistoryOperations(PEER_RECOVERY_NAME, request.startingSeqNo()) && ((retentionLeaseRef.get() == null && shard.useRetentionLeasesInPeerRecovery() == false) || (retentionLeaseRef.get() != null && retentionLeaseRef.get().retainingSequenceNumber() <= request.startingSeqNo()));
        if (isSequenceNumberBasedRecovery && retentionLeaseRef.get() != null) {
            // all the history we need is retained by an existing retention lease, so we do not need a separate retention lock
            retentionLock.close();
            logger.trace("history is retained by {}", retentionLeaseRef.get());
        } else {
            // all the history we need is retained by the retention lock, obtained before calling shard.hasCompleteHistoryOperations()
            // and before acquiring the safe commit we'll be using, so we can be certain that all operations after the safe commit's
            // local checkpoint will be retained for the duration of this recovery.
            logger.trace("history is retained by retention lock");
        }
        final StepListener<SendFileResult> sendFileStep = new StepListener<>();
        final StepListener<TimeValue> prepareEngineStep = new StepListener<>();
        final StepListener<SendSnapshotResult> sendSnapshotStep = new StepListener<>();
        final StepListener<Void> finalizeStep = new StepListener<>();
        if (isSequenceNumberBasedRecovery) {
            logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
            startingSeqNo = request.startingSeqNo();
            if (retentionLeaseRef.get() == null) {
                createRetentionLease(startingSeqNo, ActionListener.map(sendFileStep, ignored -> SendFileResult.EMPTY));
            } else {
                sendFileStep.onResponse(SendFileResult.EMPTY);
            }
        } else {
            final GatedCloseable<IndexCommit> wrappedSafeCommit;
            try {
                wrappedSafeCommit = acquireSafeCommit(shard);
                resources.add(wrappedSafeCommit);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
            }
            // Try and copy enough operations to the recovering peer so that if it is promoted to primary then it has a chance of being
            // able to recover other replicas using operations-based recoveries. If we are not using retention leases then we
            // conservatively copy all available operations. If we are using retention leases then "enough operations" is just the
            // operations from the local checkpoint of the safe commit onwards, because when using soft deletes the safe commit retains
            // at least as much history as anything else. The safe commit will often contain all the history retained by the current set
            // of retention leases, but this is not guaranteed: an earlier peer recovery from a different primary might have created a
            // retention lease for some history that this primary already discarded, since we discard history when the global checkpoint
            // advances and not when creating a new safe commit. In any case this is a best-effort thing since future recoveries can
            // always fall back to file-based ones, and only really presents a problem if this primary fails before things have settled
            // down.
            startingSeqNo = Long.parseLong(wrappedSafeCommit.get().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) + 1L;
            logger.trace("performing file-based recovery followed by history replay starting at [{}]", startingSeqNo);
            try {
                final int estimateNumOps = countNumberOfHistoryOperations(startingSeqNo);
                final Releasable releaseStore = acquireStore(shard.store());
                resources.add(releaseStore);
                sendFileStep.whenComplete(r -> IOUtils.close(wrappedSafeCommit, releaseStore), e -> {
                    try {
                        IOUtils.close(wrappedSafeCommit, releaseStore);
                    } catch (final IOException ex) {
                        logger.warn("releasing snapshot caused exception", ex);
                    }
                });
                final StepListener<ReplicationResponse> deleteRetentionLeaseStep = new StepListener<>();
                runUnderPrimaryPermit(() -> {
                    try {
                        // If the target previously had a copy of this shard then a file-based recovery might move its global
                        // checkpoint backwards. We must therefore remove any existing retention lease so that we can create a
                        // new one later on in the recovery.
                        shard.removePeerRecoveryRetentionLease(request.targetNode().getId(), new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, deleteRetentionLeaseStep, false));
                    } catch (RetentionLeaseNotFoundException e) {
                        logger.debug("no peer-recovery retention lease for " + request.targetAllocationId());
                        deleteRetentionLeaseStep.onResponse(null);
                    }
                }, shardId + " removing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger);
                deleteRetentionLeaseStep.whenComplete(ignored -> {
                    assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase1]");
                    phase1(wrappedSafeCommit.get(), startingSeqNo, () -> estimateNumOps, sendFileStep);
                }, onFailure);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "sendFileStep failed", e);
            }
        }
        assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo;
        sendFileStep.whenComplete(r -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[prepareTargetForTranslog]");
            // For a sequence based recovery, the target can keep its local translog
            prepareTargetForTranslog(countNumberOfHistoryOperations(startingSeqNo), prepareEngineStep);
        }, onFailure);
        prepareEngineStep.whenComplete(prepareEngineTime -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase2]");
            /*
                 * add shard to replication group (shard will receive replication requests from this point on) now that engine is open.
                 * This means that any document indexed into the primary after this will be replicated to this replica as well
                 * make sure to do this before sampling the max sequence number in the next step, to ensure that we send
                 * all documents up to maxSeqNo in phase2.
                 */
            runUnderPrimaryPermit(() -> shard.initiateTracking(request.targetAllocationId()), shardId + " initiating tracking of " + request.targetAllocationId(), shard, cancellableThreads, logger);
            final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
            if (logger.isTraceEnabled()) {
                logger.trace("snapshot translog for recovery; current size is [{}]", countNumberOfHistoryOperations(startingSeqNo));
            }
            final Translog.Snapshot phase2Snapshot = shard.newChangesSnapshot(PEER_RECOVERY_NAME, startingSeqNo, Long.MAX_VALUE, false, true);
            resources.add(phase2Snapshot);
            retentionLock.close();
            // we have to capture the max_seen_auto_id_timestamp and the max_seq_no_of_updates to make sure that these values
            // are at least as high as the corresponding values on the primary when any of these operations were executed on it.
            final long maxSeenAutoIdTimestamp = shard.getMaxSeenAutoIdTimestamp();
            final long maxSeqNoOfUpdatesOrDeletes = shard.getMaxSeqNoOfUpdatesOrDeletes();
            final RetentionLeases retentionLeases = shard.getRetentionLeases();
            final long mappingVersionOnPrimary = shard.indexSettings().getIndexMetadata().getMappingVersion();
            phase2(startingSeqNo, endingSeqNo, phase2Snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersionOnPrimary, sendSnapshotStep);
        }, onFailure);
        // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2
        final long trimAboveSeqNo = startingSeqNo - 1;
        sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure);
        finalizeStep.whenComplete(r -> {
            // TODO: return the actual throttle time
            final long phase1ThrottlingWaitTime = 0L;
            final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result();
            final SendFileResult sendFileResult = sendFileStep.result();
            final RecoveryResponse response = new RecoveryResponse(sendFileResult.phase1FileNames, sendFileResult.phase1FileSizes, sendFileResult.phase1ExistingFileNames, sendFileResult.phase1ExistingFileSizes, sendFileResult.totalSize, sendFileResult.existingTotalSize, sendFileResult.took.millis(), phase1ThrottlingWaitTime, prepareEngineStep.result().millis(), sendSnapshotResult.sentOperations, sendSnapshotResult.tookTime.millis());
            try {
                future.onResponse(response);
            } finally {
                IOUtils.close(resources);
            }
        }, onFailure);
    } catch (Exception e) {
        IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
    }
}

Example 3 with GatedCloseable

use of org.opensearch.common.concurrent.GatedCloseable in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSyncTranslogConcurrently.

public void testSyncTranslogConcurrently() throws Exception {
    IOUtils.close(engine, store);
    final Path translogPath = createTempDir();
    store = createStore();
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    engine = createEngine(config(defaultSettings, store, translogPath, newMergePolicy(), null, null, globalCheckpoint::get));
    List<Engine.Operation> ops = generateHistoryOnReplica(between(1, 50), false, randomBoolean(), randomBoolean());
    applyOperations(engine, ops);
    engine.flush(true, true);
    final CheckedRunnable<IOException> checker = () -> {
        assertThat(engine.getTranslogStats().getUncommittedOperations(), equalTo(0));
        assertThat(engine.getLastSyncedGlobalCheckpoint(), equalTo(globalCheckpoint.get()));
        try (GatedCloseable<IndexCommit> wrappedSafeCommit = engine.acquireSafeIndexCommit()) {
            SequenceNumbers.CommitInfo commitInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(wrappedSafeCommit.get().getUserData().entrySet());
            assertThat(commitInfo.localCheckpoint, equalTo(engine.getProcessedLocalCheckpoint()));
        }
    };
    final Thread[] threads = new Thread[randomIntBetween(2, 4)];
    final Phaser phaser = new Phaser(threads.length);
    globalCheckpoint.set(engine.getProcessedLocalCheckpoint());
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread(() -> {
            phaser.arriveAndAwaitAdvance();
            try {
                engine.syncTranslog();
                checker.run();
            } catch (IOException e) {
                throw new AssertionError(e);
            }
        });
        threads[i].start();
    }
    for (Thread thread : threads) {
        thread.join();
    }
    checker.run();
}

Example 4 with GatedCloseable

use of org.opensearch.common.concurrent.GatedCloseable in project OpenSearch by opensearch-project.

the class InternalEngineTests method testKeepMinRetainedSeqNoByMergePolicy.

public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
    IOUtils.close(engine, store);
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final long primaryTerm = randomLongBetween(1, Long.MAX_VALUE);
    final AtomicLong retentionLeasesVersion = new AtomicLong();
    final AtomicReference<RetentionLeases> retentionLeasesHolder = new AtomicReference<>(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), Collections.emptyList()));
    final List<Engine.Operation> operations = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), 2, 10, 300, "2");
    Randomness.shuffle(operations);
    Set<Long> existingSeqNos = new HashSet<>();
    store = createStore();
    engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get, retentionLeasesHolder::get));
    assertThat(engine.getMinRetainedSeqNo(), equalTo(0L));
    long lastMinRetainedSeqNo = engine.getMinRetainedSeqNo();
    for (Engine.Operation op : operations) {
        final Engine.Result result;
        if (op instanceof Engine.Index) {
            result = engine.index((Engine.Index) op);
        } else {
            result = engine.delete((Engine.Delete) op);
        }
        existingSeqNos.add(result.getSeqNo());
        if (randomBoolean()) {
            // advance persisted local checkpoint
            engine.syncTranslog();
            assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpointTracker().getPersistedCheckpoint()));
        }
        if (randomBoolean()) {
            retentionLeasesVersion.incrementAndGet();
            final int length = randomIntBetween(0, 8);
            final List<RetentionLease> leases = new ArrayList<>(length);
            for (int i = 0; i < length; i++) {
                final String id = randomAlphaOfLength(8);
                final long retainingSequenceNumber = randomLongBetween(0, Math.max(0, globalCheckpoint.get()));
                final long timestamp = randomLongBetween(0L, Long.MAX_VALUE);
                final String source = randomAlphaOfLength(8);
                leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source));
            }
            retentionLeasesHolder.set(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), leases));
        }
        if (rarely()) {
            settings.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
            indexSettings.updateIndexMetadata(IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build());
            engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
        }
        if (rarely()) {
            engine.refresh("test");
        }
        if (rarely()) {
            engine.flush(true, true);
            assertThat(Long.parseLong(engine.getLastCommittedSegmentInfos().userData.get(Engine.MIN_RETAINED_SEQNO)), equalTo(engine.getMinRetainedSeqNo()));
        }
        if (rarely()) {
            engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
        }
        try (Closeable ignored = engine.acquireHistoryRetentionLock()) {
            long minRetainSeqNos = engine.getMinRetainedSeqNo();
            assertThat(minRetainSeqNos, lessThanOrEqualTo(globalCheckpoint.get() + 1));
            Long[] expectedOps = existingSeqNos.stream().filter(seqno -> seqno >= minRetainSeqNos).toArray(Long[]::new);
            Set<Long> actualOps = readAllOperationsInLucene(engine).stream().map(Translog.Operation::seqNo).collect(Collectors.toSet());
            assertThat(actualOps, containsInAnyOrder(expectedOps));
        }
        try (GatedCloseable<IndexCommit> wrappedSafeCommit = engine.acquireSafeIndexCommit()) {
            IndexCommit safeCommit = wrappedSafeCommit.get();
            if (safeCommit.getUserData().containsKey(Engine.MIN_RETAINED_SEQNO)) {
                lastMinRetainedSeqNo = Long.parseLong(safeCommit.getUserData().get(Engine.MIN_RETAINED_SEQNO));
            }
        }
    }
    if (randomBoolean()) {
        engine.close();
    } else {
        engine.flushAndClose();
    }
    try (InternalEngine recoveringEngine = new InternalEngine(engine.config())) {
        assertThat(recoveringEngine.getMinRetainedSeqNo(), equalTo(lastMinRetainedSeqNo));
    }
}

Example 5 with GatedCloseable

use of org.opensearch.common.concurrent.GatedCloseable in project OpenSearch by opensearch-project.

the class InternalEngineTests method testConcurrentWritesAndCommits.

// this test writes documents to the engine while concurrently flushing/commit
// and ensuring that the commit points contain the correct sequence number data
public void testConcurrentWritesAndCommits() throws Exception {
    List<GatedCloseable<IndexCommit>> commits = new ArrayList<>();
    try (Store store = createStore();
        InternalEngine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
        final int numIndexingThreads = scaledRandomIntBetween(2, 4);
        final int numDocsPerThread = randomIntBetween(500, 1000);
        final CyclicBarrier barrier = new CyclicBarrier(numIndexingThreads + 1);
        final List<Thread> indexingThreads = new ArrayList<>();
        final CountDownLatch doneLatch = new CountDownLatch(numIndexingThreads);
        // create N indexing threads to index documents simultaneously
        for (int threadNum = 0; threadNum < numIndexingThreads; threadNum++) {
            final int threadIdx = threadNum;
            Thread indexingThread = new Thread(() -> {
                try {
                    // wait for all threads to start at the same time
                    barrier.await();
                    // index random number of docs
                    for (int i = 0; i < numDocsPerThread; i++) {
                        final String id = "thread" + threadIdx + "#" + i;
                        ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null);
                        engine.index(indexForDoc(doc));
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                } finally {
                    doneLatch.countDown();
                }
            });
            indexingThreads.add(indexingThread);
        }
        // start the indexing threads
        for (Thread thread : indexingThreads) {
            thread.start();
        }
        // wait for indexing threads to all be ready to start
        barrier.await();
        int commitLimit = randomIntBetween(10, 20);
        long sleepTime = 1;
        // create random commit points
        boolean doneIndexing;
        do {
            doneIndexing = doneLatch.await(sleepTime, TimeUnit.MILLISECONDS);
            commits.add(engine.acquireLastIndexCommit(true));
            if (commits.size() > commitLimit) {
                // don't keep on piling up too many commits
                IOUtils.close(commits.remove(randomIntBetween(0, commits.size() - 1)));
                // we increase the wait time to make sure we eventually if things are slow wait for threads to finish.
                // this will reduce pressure on disks and will allow threads to make progress without piling up too many commits
                sleepTime = sleepTime * 2;
            }
        } while (doneIndexing == false);
        // now, verify all the commits have the correct docs according to the user commit data
        long prevLocalCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
        long prevMaxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
        for (GatedCloseable<IndexCommit> wrappedCommit : commits) {
            final IndexCommit commit = wrappedCommit.get();
            Map<String, String> userData = commit.getUserData();
            long localCheckpoint = userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) ? Long.parseLong(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) : SequenceNumbers.NO_OPS_PERFORMED;
            long maxSeqNo = userData.containsKey(SequenceNumbers.MAX_SEQ_NO) ? Long.parseLong(userData.get(SequenceNumbers.MAX_SEQ_NO)) : UNASSIGNED_SEQ_NO;
            // local checkpoint and max seq no shouldn't go backwards
            assertThat(localCheckpoint, greaterThanOrEqualTo(prevLocalCheckpoint));
            assertThat(maxSeqNo, greaterThanOrEqualTo(prevMaxSeqNo));
            try (IndexReader reader = DirectoryReader.open(commit)) {
                Long highest = getHighestSeqNo(reader);
                final long highestSeqNo;
                if (highest != null) {
                    highestSeqNo = highest.longValue();
                } else {
                    highestSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
                }
                // make sure localCheckpoint <= highest seq no found <= maxSeqNo
                assertThat(highestSeqNo, greaterThanOrEqualTo(localCheckpoint));
                assertThat(highestSeqNo, lessThanOrEqualTo(maxSeqNo));
                // make sure all sequence numbers up to and including the local checkpoint are in the index
                FixedBitSet seqNosBitSet = getSeqNosSet(reader, highestSeqNo);
                for (int i = 0; i <= localCheckpoint; i++) {
                    assertTrue("local checkpoint [" + localCheckpoint + "], _seq_no [" + i + "] should be indexed", seqNosBitSet.get(i));
                }
            }
            prevLocalCheckpoint = localCheckpoint;
            prevMaxSeqNo = maxSeqNo;
        }
        IOUtils.close(commits);
    }
}