Search in sources :

Example 1 with RecoveryEngineException

use of org.opensearch.index.engine.RecoveryEngineException in project OpenSearch by opensearch-project.

the class RecoverySourceHandler method prepareTargetForTranslog.

void prepareTargetForTranslog(int totalTranslogOps, ActionListener<TimeValue> listener) {
    StopWatch stopWatch = new StopWatch().start();
    final ActionListener<Void> wrappedListener = ActionListener.wrap(nullVal -> {
        stopWatch.stop();
        final TimeValue tookTime = stopWatch.totalTime();
        logger.trace("recovery [phase1]: remote engine start took [{}]", tookTime);
        listener.onResponse(tookTime);
    }, e -> listener.onFailure(new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e)));
    // Send a request preparing the new shard's translog to receive operations. This ensures the shard engine is started and disables
    // garbage collection (not the JVM's GC!) of tombstone deletes.
    logger.trace("recovery [phase1]: prepare remote engine for translog");
    cancellableThreads.checkForCancel();
    recoveryTarget.prepareForTranslogOperations(totalTranslogOps, wrappedListener);
}
Also used : RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) TimeValue(org.opensearch.common.unit.TimeValue) StopWatch(org.opensearch.common.StopWatch)

Example 2 with RecoveryEngineException

use of org.opensearch.index.engine.RecoveryEngineException in project OpenSearch by opensearch-project.

the class RecoverySourceHandler method recoverToTarget.

/**
 * performs the recovery from the local engine to the target
 */
public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
    addListener(listener);
    final Closeable releaseResources = () -> IOUtils.close(resources);
    try {
        cancellableThreads.setOnCancel((reason, beforeCancelEx) -> {
            final RuntimeException e;
            if (shard.state() == IndexShardState.CLOSED) {
                // check if the shard got closed on us
                e = new IndexShardClosedException(shard.shardId(), "shard is closed and recovery was canceled reason [" + reason + "]");
            } else {
                e = new CancellableThreads.ExecutionCancelledException("recovery was canceled reason [" + reason + "]");
            }
            if (beforeCancelEx != null) {
                e.addSuppressed(beforeCancelEx);
            }
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
            throw e;
        });
        final Consumer<Exception> onFailure = e -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[onFailure]");
            IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
        };
        final SetOnce<RetentionLease> retentionLeaseRef = new SetOnce<>();
        runUnderPrimaryPermit(() -> {
            final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable();
            ShardRouting targetShardRouting = routingTable.getByAllocationId(request.targetAllocationId());
            if (targetShardRouting == null) {
                logger.debug("delaying recovery of {} as it is not listed as assigned to target node {}", request.shardId(), request.targetNode());
                throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node");
            }
            assert targetShardRouting.initializing() : "expected recovery target to be initializing but was " + targetShardRouting;
            retentionLeaseRef.set(shard.getRetentionLeases().get(ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting)));
        }, shardId + " validating recovery target [" + request.targetAllocationId() + "] registered ", shard, cancellableThreads, logger);
        final Closeable retentionLock = shard.acquireHistoryRetentionLock();
        resources.add(retentionLock);
        final long startingSeqNo;
        final boolean isSequenceNumberBasedRecovery = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && isTargetSameHistory() && shard.hasCompleteHistoryOperations(PEER_RECOVERY_NAME, request.startingSeqNo()) && ((retentionLeaseRef.get() == null && shard.useRetentionLeasesInPeerRecovery() == false) || (retentionLeaseRef.get() != null && retentionLeaseRef.get().retainingSequenceNumber() <= request.startingSeqNo()));
        if (isSequenceNumberBasedRecovery && retentionLeaseRef.get() != null) {
            // all the history we need is retained by an existing retention lease, so we do not need a separate retention lock
            retentionLock.close();
            logger.trace("history is retained by {}", retentionLeaseRef.get());
        } else {
            // all the history we need is retained by the retention lock, obtained before calling shard.hasCompleteHistoryOperations()
            // and before acquiring the safe commit we'll be using, so we can be certain that all operations after the safe commit's
            // local checkpoint will be retained for the duration of this recovery.
            logger.trace("history is retained by retention lock");
        }
        final StepListener<SendFileResult> sendFileStep = new StepListener<>();
        final StepListener<TimeValue> prepareEngineStep = new StepListener<>();
        final StepListener<SendSnapshotResult> sendSnapshotStep = new StepListener<>();
        final StepListener<Void> finalizeStep = new StepListener<>();
        if (isSequenceNumberBasedRecovery) {
            logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
            startingSeqNo = request.startingSeqNo();
            if (retentionLeaseRef.get() == null) {
                createRetentionLease(startingSeqNo, ActionListener.map(sendFileStep, ignored -> SendFileResult.EMPTY));
            } else {
                sendFileStep.onResponse(SendFileResult.EMPTY);
            }
        } else {
            final Engine.IndexCommitRef safeCommitRef;
            try {
                safeCommitRef = acquireSafeCommit(shard);
                resources.add(safeCommitRef);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
            }
            // Try and copy enough operations to the recovering peer so that if it is promoted to primary then it has a chance of being
            // able to recover other replicas using operations-based recoveries. If we are not using retention leases then we
            // conservatively copy all available operations. If we are using retention leases then "enough operations" is just the
            // operations from the local checkpoint of the safe commit onwards, because when using soft deletes the safe commit retains
            // at least as much history as anything else. The safe commit will often contain all the history retained by the current set
            // of retention leases, but this is not guaranteed: an earlier peer recovery from a different primary might have created a
            // retention lease for some history that this primary already discarded, since we discard history when the global checkpoint
            // advances and not when creating a new safe commit. In any case this is a best-effort thing since future recoveries can
            // always fall back to file-based ones, and only really presents a problem if this primary fails before things have settled
            // down.
            startingSeqNo = Long.parseLong(safeCommitRef.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) + 1L;
            logger.trace("performing file-based recovery followed by history replay starting at [{}]", startingSeqNo);
            try {
                final int estimateNumOps = estimateNumberOfHistoryOperations(startingSeqNo);
                final Releasable releaseStore = acquireStore(shard.store());
                resources.add(releaseStore);
                sendFileStep.whenComplete(r -> IOUtils.close(safeCommitRef, releaseStore), e -> {
                    try {
                        IOUtils.close(safeCommitRef, releaseStore);
                    } catch (final IOException ex) {
                        logger.warn("releasing snapshot caused exception", ex);
                    }
                });
                final StepListener<ReplicationResponse> deleteRetentionLeaseStep = new StepListener<>();
                runUnderPrimaryPermit(() -> {
                    try {
                        // If the target previously had a copy of this shard then a file-based recovery might move its global
                        // checkpoint backwards. We must therefore remove any existing retention lease so that we can create a
                        // new one later on in the recovery.
                        shard.removePeerRecoveryRetentionLease(request.targetNode().getId(), new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, deleteRetentionLeaseStep, false));
                    } catch (RetentionLeaseNotFoundException e) {
                        logger.debug("no peer-recovery retention lease for " + request.targetAllocationId());
                        deleteRetentionLeaseStep.onResponse(null);
                    }
                }, shardId + " removing retention lease for [" + request.targetAllocationId() + "]", shard, cancellableThreads, logger);
                deleteRetentionLeaseStep.whenComplete(ignored -> {
                    assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase1]");
                    phase1(safeCommitRef.getIndexCommit(), startingSeqNo, () -> estimateNumOps, sendFileStep);
                }, onFailure);
            } catch (final Exception e) {
                throw new RecoveryEngineException(shard.shardId(), 1, "sendFileStep failed", e);
            }
        }
        assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo;
        sendFileStep.whenComplete(r -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[prepareTargetForTranslog]");
            // For a sequence based recovery, the target can keep its local translog
            prepareTargetForTranslog(estimateNumberOfHistoryOperations(startingSeqNo), prepareEngineStep);
        }, onFailure);
        prepareEngineStep.whenComplete(prepareEngineTime -> {
            assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase2]");
            /*
                 * add shard to replication group (shard will receive replication requests from this point on) now that engine is open.
                 * This means that any document indexed into the primary after this will be replicated to this replica as well
                 * make sure to do this before sampling the max sequence number in the next step, to ensure that we send
                 * all documents up to maxSeqNo in phase2.
                 */
            runUnderPrimaryPermit(() -> shard.initiateTracking(request.targetAllocationId()), shardId + " initiating tracking of " + request.targetAllocationId(), shard, cancellableThreads, logger);
            final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
            if (logger.isTraceEnabled()) {
                logger.trace("snapshot translog for recovery; current size is [{}]", estimateNumberOfHistoryOperations(startingSeqNo));
            }
            final Translog.Snapshot phase2Snapshot = shard.newChangesSnapshot(PEER_RECOVERY_NAME, startingSeqNo, Long.MAX_VALUE, false);
            resources.add(phase2Snapshot);
            retentionLock.close();
            // we have to capture the max_seen_auto_id_timestamp and the max_seq_no_of_updates to make sure that these values
            // are at least as high as the corresponding values on the primary when any of these operations were executed on it.
            final long maxSeenAutoIdTimestamp = shard.getMaxSeenAutoIdTimestamp();
            final long maxSeqNoOfUpdatesOrDeletes = shard.getMaxSeqNoOfUpdatesOrDeletes();
            final RetentionLeases retentionLeases = shard.getRetentionLeases();
            final long mappingVersionOnPrimary = shard.indexSettings().getIndexMetadata().getMappingVersion();
            phase2(startingSeqNo, endingSeqNo, phase2Snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersionOnPrimary, sendSnapshotStep);
        }, onFailure);
        // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2
        final long trimAboveSeqNo = startingSeqNo - 1;
        sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure);
        finalizeStep.whenComplete(r -> {
            // TODO: return the actual throttle time
            final long phase1ThrottlingWaitTime = 0L;
            final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result();
            final SendFileResult sendFileResult = sendFileStep.result();
            final RecoveryResponse response = new RecoveryResponse(sendFileResult.phase1FileNames, sendFileResult.phase1FileSizes, sendFileResult.phase1ExistingFileNames, sendFileResult.phase1ExistingFileSizes, sendFileResult.totalSize, sendFileResult.existingTotalSize, sendFileResult.took.millis(), phase1ThrottlingWaitTime, prepareEngineStep.result().millis(), sendSnapshotResult.sentOperations, sendSnapshotResult.tookTime.millis());
            try {
                future.onResponse(response);
            } finally {
                IOUtils.close(resources);
            }
        }, onFailure);
    } catch (Exception e) {
        IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e));
    }
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) FutureUtils(org.opensearch.common.util.concurrent.FutureUtils) Releasables(org.opensearch.common.lease.Releasables) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ExceptionsHelper(org.opensearch.ExceptionsHelper) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) Store(org.opensearch.index.store.Store) Engine(org.opensearch.index.engine.Engine) List(java.util.List) Logger(org.apache.logging.log4j.Logger) BytesArray(org.opensearch.common.bytes.BytesArray) CheckedRunnable(org.opensearch.common.CheckedRunnable) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) StepListener(org.opensearch.action.StepListener) ListenableFuture(org.opensearch.common.util.concurrent.ListenableFuture) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) CancellableThreads(org.opensearch.common.util.CancellableThreads) IndexShardState(org.opensearch.index.shard.IndexShardState) BytesReference(org.opensearch.common.bytes.BytesReference) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) CompletableFuture(java.util.concurrent.CompletableFuture) Releasable(org.opensearch.common.lease.Releasable) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) ArrayList(java.util.ArrayList) IndexShard(org.opensearch.index.shard.IndexShard) Loggers(org.opensearch.common.logging.Loggers) LegacyESVersion(org.opensearch.LegacyESVersion) Translog(org.opensearch.index.translog.Translog) StreamSupport(java.util.stream.StreamSupport) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) IntSupplier(java.util.function.IntSupplier) RetentionLease(org.opensearch.index.seqno.RetentionLease) ArrayUtil(org.apache.lucene.util.ArrayUtil) StopWatch(org.opensearch.common.StopWatch) InputStreamIndexInput(org.opensearch.common.lucene.store.InputStreamIndexInput) IndexInput(org.apache.lucene.store.IndexInput) SetOnce(org.apache.lucene.util.SetOnce) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) Transports(org.opensearch.transport.Transports) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) Closeable(java.io.Closeable) Comparator(java.util.Comparator) Collections(java.util.Collections) RateLimiter(org.apache.lucene.store.RateLimiter) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Closeable(java.io.Closeable) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Translog(org.opensearch.index.translog.Translog) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) TimeValue(org.opensearch.common.unit.TimeValue) Engine(org.opensearch.index.engine.Engine) CancellableThreads(org.opensearch.common.util.CancellableThreads) SetOnce(org.apache.lucene.util.SetOnce) IOException(java.io.IOException) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) RemoteTransportException(org.opensearch.transport.RemoteTransportException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) IndexShardRelocatedException(org.opensearch.index.shard.IndexShardRelocatedException) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) RetentionLeaseNotFoundException(org.opensearch.index.seqno.RetentionLeaseNotFoundException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) RetentionLease(org.opensearch.index.seqno.RetentionLease) StepListener(org.opensearch.action.StepListener) Releasable(org.opensearch.common.lease.Releasable) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 3 with RecoveryEngineException

use of org.opensearch.index.engine.RecoveryEngineException in project OpenSearch by opensearch-project.

the class ExceptionSerializationTests method testRecoveryEngineException.

public void testRecoveryEngineException() throws IOException {
    ShardId id = new ShardId("foo", "_na_", 1);
    RecoveryEngineException ex = serialize(new RecoveryEngineException(id, 10, "total failure", new NullPointerException()));
    assertEquals(id, ex.getShardId());
    assertEquals("Phase[10] total failure", ex.getMessage());
    assertEquals(10, ex.phase());
    ex = serialize(new RecoveryEngineException(null, -1, "total failure", new NullPointerException()));
    assertNull(ex.getShardId());
    assertEquals(-1, ex.phase());
    assertTrue(ex.getCause() instanceof NullPointerException);
}
Also used : ShardId(org.opensearch.index.shard.ShardId) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException)

Example 4 with RecoveryEngineException

use of org.opensearch.index.engine.RecoveryEngineException in project OpenSearch by opensearch-project.

the class RecoverySourceHandlerTests method testSendSnapshotStopOnError.

public void testSendSnapshotStopOnError() throws Exception {
    final int fileChunkSizeInBytes = between(1, 10 * 1024);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<Translog.Operation> ops = new ArrayList<>();
    for (int numOps = between(1, 256), i = 0; i < numOps; i++) {
        final Engine.Index index = getIndex(Integer.toString(i));
        ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true)));
    }
    final AtomicBoolean wasFailed = new AtomicBoolean();
    RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp, long msu, RetentionLeases retentionLeases, long mappingVersion, ActionListener<Long> listener) {
            if (randomBoolean()) {
                listener.onResponse(SequenceNumbers.NO_OPS_PERFORMED);
            } else {
                listener.onFailure(new RuntimeException("test - failed to index"));
                wasFailed.set(true);
            }
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, new AsyncRecoveryTarget(recoveryTarget, threadPool.generic()), threadPool, request, fileChunkSizeInBytes, between(1, 10), between(1, 10));
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> future = new PlainActionFuture<>();
    final long startingSeqNo = randomLongBetween(0, ops.size() - 1L);
    final long endingSeqNo = randomLongBetween(startingSeqNo, ops.size() - 1L);
    handler.phase2(startingSeqNo, endingSeqNo, newTranslogSnapshot(ops, Collections.emptyList()), randomNonNegativeLong(), randomNonNegativeLong(), RetentionLeases.EMPTY, randomNonNegativeLong(), future);
    if (wasFailed.get()) {
        final RecoveryEngineException error = expectThrows(RecoveryEngineException.class, future::actionGet);
        assertThat(error.getMessage(), equalTo("Phase[2] failed to send/replay operations"));
        assertThat(error.getCause().getMessage(), equalTo("test - failed to index"));
    }
}
Also used : CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) Translog(org.opensearch.index.translog.Translog) RecoveryEngineException(org.opensearch.index.engine.RecoveryEngineException) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) List(java.util.List) Engine(org.opensearch.index.engine.Engine) IndexShard(org.opensearch.index.shard.IndexShard) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LatchedActionListener(org.opensearch.action.LatchedActionListener) ActionListener(org.opensearch.action.ActionListener) PlainActionFuture(org.opensearch.action.support.PlainActionFuture)

Aggregations

RecoveryEngineException (org.opensearch.index.engine.RecoveryEngineException)4 ArrayList (java.util.ArrayList)2 List (java.util.List)2 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 ActionListener (org.opensearch.action.ActionListener)2 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)2 StopWatch (org.opensearch.common.StopWatch)2 TimeValue (org.opensearch.common.unit.TimeValue)2 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 Comparator (java.util.Comparator)1 Deque (java.util.Deque)1 Locale (java.util.Locale)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ConcurrentLinkedDeque (java.util.concurrent.ConcurrentLinkedDeque)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1