Examples with ResyncTask org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask used on opensource projects

Example 1 with ResyncTask

use of org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask in project OpenSearch by opensearch-project.

the class IndexShard method updateShardState.

@Override
public void updateShardState(final ShardRouting newRouting, final long newPrimaryTerm, final BiConsumer<IndexShard, ActionListener<ResyncTask>> primaryReplicaSyncer, final long applyingClusterStateVersion, final Set<String> inSyncAllocationIds, final IndexShardRoutingTable routingTable) throws IOException {
    final ShardRouting currentRouting;
    synchronized (mutex) {
        currentRouting = this.shardRouting;
        assert currentRouting != null;
        if (!newRouting.shardId().equals(shardId())) {
            throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId());
        }
        if (newRouting.isSameAllocation(currentRouting) == false) {
            throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
        }
        if (currentRouting.primary() && newRouting.primary() == false) {
            throw new IllegalArgumentException("illegal state: trying to move shard from primary mode to replica mode. Current " + currentRouting + ", new " + newRouting);
        }
        if (newRouting.primary()) {
            replicationTracker.updateFromMaster(applyingClusterStateVersion, inSyncAllocationIds, routingTable);
        }
        if (state == IndexShardState.POST_RECOVERY && newRouting.active()) {
            assert currentRouting.active() == false : "we are in POST_RECOVERY, but our shard routing is active " + currentRouting;
            assert currentRouting.isRelocationTarget() == false || currentRouting.primary() == false || replicationTracker.isPrimaryMode() : "a primary relocation is completed by the master, but primary mode is not active " + currentRouting;
            changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
        } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isRelocated() && (newRouting.relocating() == false || newRouting.equalsIgnoringMetadata(currentRouting) == false)) {
            // reactivate primary mode without risking two active primaries.
            throw new IndexShardRelocatedException(shardId(), "Shard is marked as relocated, cannot safely move to state " + newRouting.state());
        }
        assert newRouting.active() == false || state == IndexShardState.STARTED || state == IndexShardState.CLOSED : "routing is active, but local shard state isn't. routing: " + newRouting + ", local state: " + state;
        persistMetadata(path, indexSettings, newRouting, currentRouting, logger);
        final CountDownLatch shardStateUpdated = new CountDownLatch(1);
        if (newRouting.primary()) {
            if (newPrimaryTerm == pendingPrimaryTerm) {
                if (currentRouting.initializing() && currentRouting.isRelocationTarget() == false && newRouting.active()) {
                    // the master started a recovering primary, activate primary mode.
                    replicationTracker.activatePrimaryMode(getLocalCheckpoint());
                    ensurePeerRecoveryRetentionLeasesExist();
                }
            } else {
                assert currentRouting.primary() == false : "term is only increased as part of primary promotion";
                /* Note that due to cluster state batching an initializing primary shard term can failed and re-assigned
                     * in one state causing it's term to be incremented. Note that if both current shard state and new
                     * shard state are initializing, we could replace the current shard and reinitialize it. It is however
                     * possible that this shard is being started. This can happen if:
                     * 1) Shard is post recovery and sends shard started to the master
                     * 2) Node gets disconnected and rejoins
                     * 3) Master assigns the shard back to the node
                     * 4) Master processes the shard started and starts the shard
                     * 5) The node process the cluster state where the shard is both started and primary term is incremented.
                     *
                     * We could fail the shard in that case, but this will cause it to be removed from the insync allocations list
                     * potentially preventing re-allocation.
                     */
                assert newRouting.initializing() == false : "a started primary shard should never update its term; " + "shard " + newRouting + ", " + "current term [" + pendingPrimaryTerm + "], " + "new term [" + newPrimaryTerm + "]";
                assert newPrimaryTerm > pendingPrimaryTerm : "primary terms can only go up; current term [" + pendingPrimaryTerm + "], new term [" + newPrimaryTerm + "]";
                /*
                     * Before this call returns, we are guaranteed that all future operations are delayed and so this happens before we
                     * increment the primary term. The latch is needed to ensure that we do not unblock operations before the primary
                     * term is incremented.
                     */
                // to prevent primary relocation handoff while resync is not completed
                boolean resyncStarted = primaryReplicaResyncInProgress.compareAndSet(false, true);
                if (resyncStarted == false) {
                    throw new IllegalStateException("cannot start resync while it's already in progress");
                }
                bumpPrimaryTerm(newPrimaryTerm, () -> {
                    shardStateUpdated.await();
                    assert pendingPrimaryTerm == newPrimaryTerm : "shard term changed on primary. expected [" + newPrimaryTerm + "] but was [" + pendingPrimaryTerm + "]" + ", current routing: " + currentRouting + ", new routing: " + newRouting;
                    assert getOperationPrimaryTerm() == newPrimaryTerm;
                    try {
                        replicationTracker.activatePrimaryMode(getLocalCheckpoint());
                        ensurePeerRecoveryRetentionLeasesExist();
                        /*
                             * If this shard was serving as a replica shard when another shard was promoted to primary then
                             * its Lucene index was reset during the primary term transition. In particular, the Lucene index
                             * on this shard was reset to the global checkpoint and the operations above the local checkpoint
                             * were reverted. If the other shard that was promoted to primary subsequently fails before the
                             * primary/replica re-sync completes successfully and we are now being promoted, we have to restore
                             * the reverted operations on this shard by replaying the translog to avoid losing acknowledged writes.
                             */
                        final Engine engine = getEngine();
                        engine.restoreLocalHistoryFromTranslog((resettingEngine, snapshot) -> runTranslogRecovery(resettingEngine, snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
                        }));
                        /* Rolling the translog generation is not strictly needed here (as we will never have collisions between
                             * sequence numbers in a translog generation in a new primary as it takes the last known sequence number
                             * as a starting point), but it simplifies reasoning about the relationship between primary terms and
                             * translog generations.
                             */
                        engine.rollTranslogGeneration();
                        engine.fillSeqNoGaps(newPrimaryTerm);
                        replicationTracker.updateLocalCheckpoint(currentRouting.allocationId().getId(), getLocalCheckpoint());
                        primaryReplicaSyncer.accept(this, new ActionListener<ResyncTask>() {

                            @Override
                            public void onResponse(ResyncTask resyncTask) {
                                logger.info("primary-replica resync completed with {} operations", resyncTask.getResyncedOperations());
                                boolean resyncCompleted = primaryReplicaResyncInProgress.compareAndSet(true, false);
                                assert resyncCompleted : "primary-replica resync finished but was not started";
                            }

                            @Override
                            public void onFailure(Exception e) {
                                boolean resyncCompleted = primaryReplicaResyncInProgress.compareAndSet(true, false);
                                assert resyncCompleted : "primary-replica resync finished but was not started";
                                if (state == IndexShardState.CLOSED) {
                                // ignore, shutting down
                                } else {
                                    failShard("exception during primary-replica resync", e);
                                }
                            }
                        });
                    } catch (final AlreadyClosedException e) {
                    // okay, the index was deleted
                    }
                }, null);
            }
        }
        // set this last, once we finished updating all internal state.
        this.shardRouting = newRouting;
        assert // note that we use started and not
        this.shardRouting.primary() == false || this.shardRouting.started() == false || // active to avoid relocating shards
        this.indexShardOperationPermits.isBlocked() || this.replicationTracker.isPrimaryMode() : "a started primary with non-pending operation term must be in primary mode " + this.shardRouting;
        shardStateUpdated.countDown();
    }
    if (currentRouting.active() == false && newRouting.active()) {
        indexEventListener.afterIndexShardStarted(this);
    }
    if (newRouting.equals(currentRouting) == false) {
        indexEventListener.shardRoutingChanged(this, currentRouting, newRouting);
    }
    if (indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery == false && state() == IndexShardState.STARTED) {
        final RetentionLeases retentionLeases = replicationTracker.getRetentionLeases();
        final Set<ShardRouting> shardRoutings = new HashSet<>(routingTable.getShards());
        // include relocation targets
        shardRoutings.addAll(routingTable.assignedShards());
        if (shardRoutings.stream().allMatch(shr -> shr.assignedToNode() && retentionLeases.contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(shr)))) {
            useRetentionLeasesInPeerRecovery = true;
            turnOffTranslogRetention();
        }
    }
}