Search in sources :

Example 11 with RetentionLease

use of org.opensearch.index.seqno.RetentionLease in project OpenSearch by opensearch-project.

the class RecoverySourceHandlerTests method testCancelRecoveryDuringPhase1.

public void testCancelRecoveryDuringPhase1() throws Exception {
    Store store = newStore(createTempDir("source"), false);
    IndexShard shard = mock(IndexShard.class);
    when(shard.store()).thenReturn(store);
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    AtomicBoolean wasCancelled = new AtomicBoolean();
    SetOnce<Runnable> cancelRecovery = new SetOnce<>();
    final TestRecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        @Override
        public void receiveFileInfo(List<String> phase1FileNames, List<Long> phase1FileSizes, List<String> phase1ExistingFileNames, List<Long> phase1ExistingFileSizes, int totalTranslogOps, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (randomBoolean()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (rarely()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }

        @Override
        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetadata, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (randomBoolean()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }
    };
    final StartRecoveryRequest startRecoveryRequest = getStartRecoveryRequest();
    final RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, threadPool, startRecoveryRequest, between(1, 16), between(1, 4), between(1, 4)) {

        @Override
        void createRetentionLease(long startingSeqNo, ActionListener<RetentionLease> listener) {
            final String leaseId = ReplicationTracker.getPeerRecoveryRetentionLeaseId(startRecoveryRequest.targetNode().getId());
            listener.onResponse(new RetentionLease(leaseId, startingSeqNo, threadPool.absoluteTimeInMillis(), ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE));
        }
    };
    cancelRecovery.set(() -> handler.cancel("test"));
    final StepListener<RecoverySourceHandler.SendFileResult> phase1Listener = new StepListener<>();
    try {
        final CountDownLatch latch = new CountDownLatch(1);
        handler.phase1(DirectoryReader.listCommits(dir).get(0), 0, () -> 0, new LatchedActionListener<>(phase1Listener, latch));
        latch.await();
        phase1Listener.result();
    } catch (Exception e) {
        assertTrue(wasCancelled.get());
        assertNotNull(ExceptionsHelper.unwrap(e, CancellableThreads.ExecutionCancelledException.class));
    }
    store.close();
}
Also used : Store(org.opensearch.index.store.Store) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) Matchers.containsString(org.hamcrest.Matchers.containsString) Mockito.anyString(org.mockito.Mockito.anyString) Document(org.apache.lucene.document.Document) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) List(java.util.List) Directory(org.apache.lucene.store.Directory) BytesReference(org.opensearch.common.bytes.BytesReference) CancellableThreads(org.opensearch.common.util.CancellableThreads) SetOnce(org.apache.lucene.util.SetOnce) IndexShard(org.opensearch.index.shard.IndexShard) CountDownLatch(java.util.concurrent.CountDownLatch) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) IOException(java.io.IOException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LatchedActionListener(org.opensearch.action.LatchedActionListener) ActionListener(org.opensearch.action.ActionListener) RetentionLease(org.opensearch.index.seqno.RetentionLease) StringField(org.apache.lucene.document.StringField) StepListener(org.opensearch.action.StepListener) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter)

Example 12 with RetentionLease

use of org.opensearch.index.seqno.RetentionLease in project OpenSearch by opensearch-project.

the class RecoverySourceHandler method createRetentionLease.

void createRetentionLease(final long startingSeqNo, ActionListener<RetentionLease> listener) {
    runUnderPrimaryPermit(() -> {
        // Clone the peer recovery retention lease belonging to the source shard. We are retaining history between the the local
        // checkpoint of the safe commit we're creating and this lease's retained seqno with the retention lock, and by cloning an
        // existing lease we (approximately) know that all our peers are also retaining history as requested by the cloned lease. If
        // the recovery now fails before copying enough history over then a subsequent attempt will find this lease, determine it is
        // not enough, and fall back to a file-based recovery.
        // 
        // (approximately) because we do not guarantee to be able to satisfy every lease on every peer.
        logger.trace("cloning primary's retention lease");
        try {
            final StepListener<ReplicationResponse> cloneRetentionLeaseStep = new StepListener<>();
            final RetentionLease clonedLease = shard.cloneLocalPeerRecoveryRetentionLease(request.targetNode().getId(), new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false));
            logger.trace("cloned primary's retention lease as [{}]", clonedLease);
            cloneRetentionLeaseStep.whenComplete(rr -> listener.onResponse(clonedLease), listener::onFailure);
        } catch (RetentionLeaseNotFoundException e) {
            // recovery as a conservative estimate for the global checkpoint.
            assert shard.indexSettings().getIndexVersionCreated().before(LegacyESVersion.V_7_4_0) || shard.indexSettings().isSoftDeleteEnabled() == false;
            final StepListener<ReplicationResponse> addRetentionLeaseStep = new StepListener<>();
            final long estimatedGlobalCheckpoint = startingSeqNo - 1;
            final RetentionLease newLease = shard.addPeerRecoveryRetentionLease(request.targetNode().getId(), estimatedGlobalCheckpoint, new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false));
            addRetentionLeaseStep.whenComplete(rr -> listener.onResponse(newLease), listener::onFailure);
            logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint);
        }
    }, shardId + " establishing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger);
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) FutureUtils(org.opensearch.common.util.concurrent.FutureUtils) Releasables(org.opensearch.common.lease.Releasables) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ExceptionsHelper(org.opensearch.ExceptionsHelper) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) Store(org.opensearch.index.store.Store) Engine(org.opensearch.index.engine.Engine) List(java.util.List) Logger(org.apache.logging.log4j.Logger) BytesArray(org.opensearch.common.bytes.BytesArray) CheckedRunnable(org.opensearch.common.CheckedRunnable) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) StepListener(org.opensearch.action.StepListener) ListenableFuture(org.opensearch.common.util.concurrent.ListenableFuture) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) CancellableThreads(org.opensearch.common.util.CancellableThreads) IndexShardState(org.opensearch.index.shard.IndexShardState) BytesReference(org.opensearch.common.bytes.BytesReference) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) CompletableFuture(java.util.concurrent.CompletableFuture) Releasable(org.opensearch.common.lease.Releasable) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) ArrayList(java.util.ArrayList) IndexShard(org.opensearch.index.shard.IndexShard) Loggers(org.opensearch.common.logging.Loggers) LegacyESVersion(org.opensearch.LegacyESVersion) Translog(org.opensearch.index.translog.Translog) StreamSupport(java.util.stream.StreamSupport) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) IntSupplier(java.util.function.IntSupplier) RetentionLease(org.opensearch.index.seqno.RetentionLease) ArrayUtil(org.apache.lucene.util.ArrayUtil) StopWatch(org.opensearch.common.StopWatch) InputStreamIndexInput(org.opensearch.common.lucene.store.InputStreamIndexInput) IndexInput(org.apache.lucene.store.IndexInput) SetOnce(org.apache.lucene.util.SetOnce) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) Transports(org.opensearch.transport.Transports) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) Closeable(java.io.Closeable) Comparator(java.util.Comparator) Collections(java.util.Collections) RateLimiter(org.apache.lucene.store.RateLimiter) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) RetentionLease(org.opensearch.index.seqno.RetentionLease) StepListener(org.opensearch.action.StepListener) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse)

Example 13 with RetentionLease

use of org.opensearch.index.seqno.RetentionLease in project OpenSearch by opensearch-project.

the class RecoverySourceHandler method recoverToTarget.

/**
 * performs the recovery from the local engine to the target
 */
public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
    addListener(listener);
    final Closeable releaseResources = () -> IOUtils.close(resources);
    try {
        cancellableThreads.setOnCancel((reason, beforeCancelEx) -> {
            final RuntimeException e;
            if (shard.state() == IndexShardState.CLOSED) {
                // check if the shard got closed on us
                e = new IndexShardClosedException(shard.shardId(), "shard is closed and recovery was canceled reason [" + reason + "]");
            } else {
                e = new CancellableThreads.ExecutionCancelledException("recovery was canceled reason [" + reason + "]");
            }
            if (beforeCancelEx != null) {
                e.addSuppressed(beforeCancelEx);
            }
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
            throw e;
        });
        final Consumer<Exception> onFailure = e -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[onFailure]");
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
        };
        final SetOnce<RetentionLease> retentionLeaseRef = new SetOnce<>();
        runUnderPrimaryPermit(() -> {
            final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable();
            ShardRouting targetShardRouting = routingTable.getByAllocationId(request.targetAllocationId());
            if (targetShardRouting == null) {
                logger.debug("delaying recovery of {} as it is not listed as assigned to target node {}", request.shardId(), request.targetNode());
                throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node");
            }
            assert targetShardRouting.initializing() : "expected recovery target to be initializing but was " + targetShardRouting;
            retentionLeaseRef.set(shard.getRetentionLeases().get(ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting)));
        }, shardId + " validating recovery target [" + request.targetAllocationId() + "] registered ", shard, cancellableThreads, logger);
        final Closeable retentionLock = shard.acquireHistoryRetentionLock();
        resources.add(retentionLock);
        final long startingSeqNo;
        final boolean isSequenceNumberBasedRecovery = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && isTargetSameHistory() && shard.hasCompleteHistoryOperations(PEER_RECOVERY_NAME, request.startingSeqNo()) && ((retentionLeaseRef.get() == null && shard.useRetentionLeasesInPeerRecovery() == false) || (retentionLeaseRef.get() != null && retentionLeaseRef.get().retainingSequenceNumber() <= request.startingSeqNo()));
        if (isSequenceNumberBasedRecovery && retentionLeaseRef.get() != null) {
            // all the history we need is retained by an existing retention lease, so we do not need a separate retention lock
            retentionLock.close();
            logger.trace("history is retained by {}", retentionLeaseRef.get());
        } else {
            // all the history we need is retained by the retention lock, obtained before calling shard.hasCompleteHistoryOperations()
            // and before acquiring the safe commit we'll be using, so we can be certain that all operations after the safe commit's
            // local checkpoint will be retained for the duration of this recovery.
            logger.trace("history is retained by retention lock");
        }
        final StepListener<SendFileResult> sendFileStep = new StepListener<>();
        final StepListener<TimeValue> prepareEngineStep = new StepListener<>();
        final StepListener<SendSnapshotResult> sendSnapshotStep = new StepListener<>();
        final StepListener<Void> finalizeStep = new StepListener<>();
        if (isSequenceNumberBasedRecovery) {
            logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
            startingSeqNo = request.startingSeqNo();
            if (retentionLeaseRef.get() == null) {
                createRetentionLease(startingSeqNo, ActionListener.map(sendFileStep, ignored -> SendFileResult.EMPTY));
            } else {
                sendFileStep.onResponse(SendFileResult.EMPTY);
            }
        } else {
            final Engine.IndexCommitRef safeCommitRef;
            try {
                safeCommitRef = acquireSafeCommit(shard);
                resources.add(safeCommitRef);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
            }
            // Try and copy enough operations to the recovering peer so that if it is promoted to primary then it has a chance of being
            // able to recover other replicas using operations-based recoveries. If we are not using retention leases then we
            // conservatively copy all available operations. If we are using retention leases then "enough operations" is just the
            // operations from the local checkpoint of the safe commit onwards, because when using soft deletes the safe commit retains
            // at least as much history as anything else. The safe commit will often contain all the history retained by the current set
            // of retention leases, but this is not guaranteed: an earlier peer recovery from a different primary might have created a
            // retention lease for some history that this primary already discarded, since we discard history when the global checkpoint
            // advances and not when creating a new safe commit. In any case this is a best-effort thing since future recoveries can
            // always fall back to file-based ones, and only really presents a problem if this primary fails before things have settled
            // down.
            startingSeqNo = Long.parseLong(safeCommitRef.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) + 1L;
            logger.trace("performing file-based recovery followed by history replay starting at [{}]", startingSeqNo);
            try {
                final int estimateNumOps = estimateNumberOfHistoryOperations(startingSeqNo);
                final Releasable releaseStore = acquireStore(shard.store());
                resources.add(releaseStore);
                sendFileStep.whenComplete(r -> IOUtils.close(safeCommitRef, releaseStore), e -> {
                    try {
                        IOUtils.close(safeCommitRef, releaseStore);
                    } catch (final IOException ex) {
                        logger.warn("releasing snapshot caused exception", ex);
                    }
                });
                final StepListener<ReplicationResponse> deleteRetentionLeaseStep = new StepListener<>();
                runUnderPrimaryPermit(() -> {
                    try {
                        // If the target previously had a copy of this shard then a file-based recovery might move its global
                        // checkpoint backwards. We must therefore remove any existing retention lease so that we can create a
                        // new one later on in the recovery.
                        shard.removePeerRecoveryRetentionLease(request.targetNode().getId(), new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, deleteRetentionLeaseStep, false));
                    } catch (RetentionLeaseNotFoundException e) {
                        logger.debug("no peer-recovery retention lease for " + request.targetAllocationId());
                        deleteRetentionLeaseStep.onResponse(null);
                    }
                }, shardId + " removing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger);
                deleteRetentionLeaseStep.whenComplete(ignored -> {
                    assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase1]");
                    phase1(safeCommitRef.getIndexCommit(), startingSeqNo, () -> estimateNumOps, sendFileStep);
                }, onFailure);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "sendFileStep failed", e);
            }
        }
        assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo;
        sendFileStep.whenComplete(r -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[prepareTargetForTranslog]");
            // For a sequence based recovery, the target can keep its local translog
            prepareTargetForTranslog(estimateNumberOfHistoryOperations(startingSeqNo), prepareEngineStep);
        }, onFailure);
        prepareEngineStep.whenComplete(prepareEngineTime -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase2]");
            /*
                 * add shard to replication group (shard will receive replication requests from this point on) now that engine is open.
                 * This means that any document indexed into the primary after this will be replicated to this replica as well
                 * make sure to do this before sampling the max sequence number in the next step, to ensure that we send
                 * all documents up to maxSeqNo in phase2.
                 */
            runUnderPrimaryPermit(() -> shard.initiateTracking(request.targetAllocationId()), shardId + " initiating tracking of " + request.targetAllocationId(), shard, cancellableThreads, logger);
            final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
            if (logger.isTraceEnabled()) {
                logger.trace("snapshot translog for recovery; current size is [{}]", estimateNumberOfHistoryOperations(startingSeqNo));
            }
            final Translog.Snapshot phase2Snapshot = shard.newChangesSnapshot(PEER_RECOVERY_NAME, startingSeqNo, Long.MAX_VALUE, false);
            resources.add(phase2Snapshot);
            retentionLock.close();
            // we have to capture the max_seen_auto_id_timestamp and the max_seq_no_of_updates to make sure that these values
            // are at least as high as the corresponding values on the primary when any of these operations were executed on it.
            final long maxSeenAutoIdTimestamp = shard.getMaxSeenAutoIdTimestamp();
            final long maxSeqNoOfUpdatesOrDeletes = shard.getMaxSeqNoOfUpdatesOrDeletes();
            final RetentionLeases retentionLeases = shard.getRetentionLeases();
            final long mappingVersionOnPrimary = shard.indexSettings().getIndexMetadata().getMappingVersion();
            phase2(startingSeqNo, endingSeqNo, phase2Snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersionOnPrimary, sendSnapshotStep);
        }, onFailure);
        // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2
        final long trimAboveSeqNo = startingSeqNo - 1;
        sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure);
        finalizeStep.whenComplete(r -> {
            // TODO: return the actual throttle time
            final long phase1ThrottlingWaitTime = 0L;
            final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result();
            final SendFileResult sendFileResult = sendFileStep.result();
            final RecoveryResponse response = new RecoveryResponse(sendFileResult.phase1FileNames, sendFileResult.phase1FileSizes, sendFileResult.phase1ExistingFileNames, sendFileResult.phase1ExistingFileSizes, sendFileResult.totalSize, sendFileResult.existingTotalSize, sendFileResult.took.millis(), phase1ThrottlingWaitTime, prepareEngineStep.result().millis(), sendSnapshotResult.sentOperations, sendSnapshotResult.tookTime.millis());
            try {
                future.onResponse(response);
            } finally {
                IOUtils.close(resources);
            }
        }, onFailure);
    } catch (Exception e) {
        IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
    }
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) FutureUtils(org.opensearch.common.util.concurrent.FutureUtils) Releasables(org.opensearch.common.lease.Releasables) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ExceptionsHelper(org.opensearch.ExceptionsHelper) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) Store(org.opensearch.index.store.Store) Engine(org.opensearch.index.engine.Engine) List(java.util.List) Logger(org.apache.logging.log4j.Logger) BytesArray(org.opensearch.common.bytes.BytesArray) CheckedRunnable(org.opensearch.common.CheckedRunnable) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) StepListener(org.opensearch.action.StepListener) ListenableFuture(org.opensearch.common.util.concurrent.ListenableFuture) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) CancellableThreads(org.opensearch.common.util.CancellableThreads) IndexShardState(org.opensearch.index.shard.IndexShardState) BytesReference(org.opensearch.common.bytes.BytesReference) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) CompletableFuture(java.util.concurrent.CompletableFuture) Releasable(org.opensearch.common.lease.Releasable) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) ArrayList(java.util.ArrayList) IndexShard(org.opensearch.index.shard.IndexShard) Loggers(org.opensearch.common.logging.Loggers) LegacyESVersion(org.opensearch.LegacyESVersion) Translog(org.opensearch.index.translog.Translog) StreamSupport(java.util.stream.StreamSupport) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) IntSupplier(java.util.function.IntSupplier) RetentionLease(org.opensearch.index.seqno.RetentionLease) ArrayUtil(org.apache.lucene.util.ArrayUtil) StopWatch(org.opensearch.common.StopWatch) InputStreamIndexInput(org.opensearch.common.lucene.store.InputStreamIndexInput) IndexInput(org.apache.lucene.store.IndexInput) SetOnce(org.apache.lucene.util.SetOnce) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) Transports(org.opensearch.transport.Transports) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) Closeable(java.io.Closeable) Comparator(java.util.Comparator) Collections(java.util.Collections) RateLimiter(org.apache.lucene.store.RateLimiter) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Closeable(java.io.Closeable) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Translog(org.opensearch.index.translog.Translog) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) TimeValue(org.opensearch.common.unit.TimeValue) Engine(org.opensearch.index.engine.Engine) CancellableThreads(org.opensearch.common.util.CancellableThreads) SetOnce(org.apache.lucene.util.SetOnce) IOException(java.io.IOException) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) RemoteTransportException(org.opensearch.transport.RemoteTransportException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) RetentionLease(org.opensearch.index.seqno.RetentionLease) StepListener(org.opensearch.action.StepListener) Releasable(org.opensearch.common.lease.Releasable) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 14 with RetentionLease

use of org.opensearch.index.seqno.RetentionLease in project OpenSearch by opensearch-project.

the class ReplicaShardAllocatorTests method testNotCancellingRecoveryIfCurrentRecoveryHasRetentionLease.

public void testNotCancellingRecoveryIfCurrentRecoveryHasRetentionLease() {
    RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders());
    List<RetentionLease> peerRecoveryRetentionLeasesOnPrimary = new ArrayList<>();
    long retainingSeqNo = randomLongBetween(1, Long.MAX_VALUE);
    peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node1, retainingSeqNo));
    peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node2, randomLongBetween(1, retainingSeqNo)));
    if (randomBoolean()) {
        peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node3, randomLongBetween(0, retainingSeqNo)));
    }
    testAllocator.addData(node1, peerRecoveryRetentionLeasesOnPrimary, "MATCH", new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.addData(node2, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.addData(node3, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.processExistingRecoveries(allocation);
    assertThat(allocation.routingNodesChanged(), equalTo(false));
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(0));
}
Also used : RetentionLease(org.opensearch.index.seqno.RetentionLease) ArrayList(java.util.ArrayList) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) RoutingAllocation(org.opensearch.cluster.routing.allocation.RoutingAllocation)

Example 15 with RetentionLease

use of org.opensearch.index.seqno.RetentionLease in project OpenSearch by opensearch-project.

the class ReplicaShardAllocatorTests method testIgnoreRetentionLeaseIfCopyIsEmpty.

public void testIgnoreRetentionLeaseIfCopyIsEmpty() {
    RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
    long retainingSeqNo = randomLongBetween(1, Long.MAX_VALUE);
    List<RetentionLease> retentionLeases = new ArrayList<>();
    retentionLeases.add(newRetentionLease(node1, retainingSeqNo));
    retentionLeases.add(newRetentionLease(node2, randomLongBetween(0, retainingSeqNo)));
    if (randomBoolean()) {
        retentionLeases.add(newRetentionLease(node3, randomLongBetween(0, retainingSeqNo)));
    }
    testAllocator.addData(node1, retentionLeases, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    // has retention lease but store is empty
    testAllocator.addData(node2, null);
    testAllocator.addData(node3, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    allocateAllUnassigned(allocation);
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node3.getId()));
}
Also used : RetentionLease(org.opensearch.index.seqno.RetentionLease) ArrayList(java.util.ArrayList) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) RoutingAllocation(org.opensearch.cluster.routing.allocation.RoutingAllocation)

Aggregations

RetentionLease (org.opensearch.index.seqno.RetentionLease)23 ArrayList (java.util.ArrayList)16 RetentionLeases (org.opensearch.index.seqno.RetentionLeases)11 AtomicLong (java.util.concurrent.atomic.AtomicLong)9 StoreFileMetadata (org.opensearch.index.store.StoreFileMetadata)9 IOException (java.io.IOException)7 List (java.util.List)7 CountDownLatch (java.util.concurrent.CountDownLatch)7 Settings (org.opensearch.common.settings.Settings)7 IndexSettings (org.opensearch.index.IndexSettings)7 Arrays (java.util.Arrays)6 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 IndexShard (org.opensearch.index.shard.IndexShard)6 LongPoint (org.apache.lucene.document.LongPoint)5 CorruptIndexException (org.apache.lucene.index.CorruptIndexException)5 ActionListener (org.opensearch.action.ActionListener)5 TimeValue (org.opensearch.common.unit.TimeValue)5 ReplicationTracker (org.opensearch.index.seqno.ReplicationTracker)5 Store (org.opensearch.index.store.Store)5 Collections (java.util.Collections)4