Examples with ResyncTask org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask used on opensource projects

Example 1 with ResyncTask

use of org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask in project crate by crate.

the class IndexShard method updateShardState.

@Override
public void updateShardState(final ShardRouting newRouting, final long newPrimaryTerm, final BiConsumer<IndexShard, ActionListener<ResyncTask>> primaryReplicaSyncer, final long applyingClusterStateVersion, final Set<String> inSyncAllocationIds, final IndexShardRoutingTable routingTable) throws IOException {
    final ShardRouting currentRouting;
    synchronized (mutex) {
        currentRouting = this.shardRouting;
        assert currentRouting != null : "shardRouting must not be null";
        if (!newRouting.shardId().equals(shardId())) {
            throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId());
        }
        if (newRouting.isSameAllocation(currentRouting) == false) {
            throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
        }
        if (currentRouting.primary() && newRouting.primary() == false) {
            throw new IllegalArgumentException("illegal state: trying to move shard from primary mode to replica mode. Current " + currentRouting + ", new " + newRouting);
        }
        if (newRouting.primary()) {
            replicationTracker.updateFromMaster(applyingClusterStateVersion, inSyncAllocationIds, routingTable);
        }
        if (state == IndexShardState.POST_RECOVERY && newRouting.active()) {
            assert currentRouting.active() == false : "we are in POST_RECOVERY, but our shard routing is active " + currentRouting;
            assert currentRouting.isRelocationTarget() == false || currentRouting.primary() == false || replicationTracker.isPrimaryMode() : "a primary relocation is completed by the master, but primary mode is not active " + currentRouting;
            changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
        } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isRelocated() && (newRouting.relocating() == false || newRouting.equalsIgnoringMetadata(currentRouting) == false)) {
            // active primaries.
            throw new IndexShardRelocatedException(shardId(), "Shard is marked as relocated, cannot safely move to state " + newRouting.state());
        }
        assert newRouting.active() == false || state == IndexShardState.STARTED || state == IndexShardState.CLOSED : "routing is active, but local shard state isn't. routing: " + newRouting + ", local state: " + state;
        persistMetadata(path, indexSettings, newRouting, currentRouting, logger);
        final CountDownLatch shardStateUpdated = new CountDownLatch(1);
        if (newRouting.primary()) {
            if (newPrimaryTerm == pendingPrimaryTerm) {
                if (currentRouting.initializing() && newRouting.active()) {
                    if (currentRouting.isRelocationTarget() == false) {
                        // the master started a recovering primary, activate primary mode.
                        replicationTracker.activatePrimaryMode(getLocalCheckpoint());
                    }
                }
            } else {
                assert currentRouting.primary() == false : "term is only increased as part of primary promotion";
                /* Note that due to cluster state batching an initializing primary shard term can failed and re-assigned
                     * in one state causing it's term to be incremented. Note that if both current shard state and new
                     * shard state are initializing, we could replace the current shard and reinitialize it. It is however
                     * possible that this shard is being started. This can happen if:
                     * 1) Shard is post recovery and sends shard started to the master
                     * 2) Node gets disconnected and rejoins
                     * 3) Master assigns the shard back to the node
                     * 4) Master processes the shard started and starts the shard
                     * 5) The node process the cluster state where the shard is both started and primary term is incremented.
                     *
                     * We could fail the shard in that case, but this will cause it to be removed from the insync allocations list
                     * potentially preventing re-allocation.
                     */
                assert newRouting.initializing() == false : "a started primary shard should never update its term; " + "shard " + newRouting + ", " + "current term [" + pendingPrimaryTerm + "], " + "new term [" + newPrimaryTerm + "]";
                assert newPrimaryTerm > pendingPrimaryTerm : "primary terms can only go up; current term [" + pendingPrimaryTerm + "], new term [" + newPrimaryTerm + "]";
                /*
                     * Before this call returns, we are guaranteed that all future operations are delayed and so this happens before we
                     * increment the primary term. The latch is needed to ensure that we do not unblock operations before the primary term is
                     * incremented.
                     */
                // to prevent primary relocation handoff while resync is not completed
                boolean resyncStarted = primaryReplicaResyncInProgress.compareAndSet(false, true);
                if (resyncStarted == false) {
                    throw new IllegalStateException("cannot start resync while it's already in progress");
                }
                bumpPrimaryTerm(newPrimaryTerm, () -> {
                    shardStateUpdated.await();
                    assert pendingPrimaryTerm == newPrimaryTerm : "shard term changed on primary. expected [" + newPrimaryTerm + "] but was [" + pendingPrimaryTerm + "]" + ", current routing: " + currentRouting + ", new routing: " + newRouting;
                    assert getOperationPrimaryTerm() == newPrimaryTerm;
                    try {
                        replicationTracker.activatePrimaryMode(getLocalCheckpoint());
                        ensurePeerRecoveryRetentionLeasesExist();
                        /*
                                 * If this shard was serving as a replica shard when another shard was promoted to primary then
                                 * its Lucene index was reset during the primary term transition. In particular, the Lucene index
                                 * on this shard was reset to the global checkpoint and the operations above the local checkpoint
                                 * were reverted. If the other shard that was promoted to primary subsequently fails before the
                                 * primary/replica re-sync completes successfully and we are now being promoted, we have to restore
                                 * the reverted operations on this shard by replaying the translog to avoid losing acknowledged writes.
                                 */
                        final Engine engine = getEngine();
                        engine.restoreLocalHistoryFromTranslog((resettingEngine, snapshot) -> runTranslogRecovery(resettingEngine, snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
                        }));
                        if (indexSettings.getIndexVersionCreated().onOrBefore(Version.V_3_0_1)) {
                            // an index that was created before sequence numbers were introduced may contain operations in its
                            // translog that do not have a sequence numbers. We want to make sure those operations will never
                            // be replayed as part of peer recovery to avoid an arbitrary mixture of operations with seq# (due
                            // to active indexing) and operations without a seq# coming from the translog. We therefore flush
                            // to create a lucene commit point to an empty translog file.
                            engine.flush(false, true);
                        }
                        /* Rolling the translog generation is not strictly needed here (as we will never have collisions between
                                 * sequence numbers in a translog generation in a new primary as it takes the last known sequence number
                                 * as a starting point), but it simplifies reasoning about the relationship between primary terms and
                                 * translog generations.
                                 */
                        engine.rollTranslogGeneration();
                        engine.fillSeqNoGaps(newPrimaryTerm);
                        replicationTracker.updateLocalCheckpoint(currentRouting.allocationId().getId(), getLocalCheckpoint());
                        primaryReplicaSyncer.accept(this, new ActionListener<ResyncTask>() {

                            @Override
                            public void onResponse(ResyncTask resyncTask) {
                                logger.info("primary-replica resync completed with {} operations", resyncTask.getResyncedOperations());
                                boolean resyncCompleted = primaryReplicaResyncInProgress.compareAndSet(true, false);
                                assert resyncCompleted : "primary-replica resync finished but was not started";
                            }

                            @Override
                            public void onFailure(Exception e) {
                                boolean resyncCompleted = primaryReplicaResyncInProgress.compareAndSet(true, false);
                                assert resyncCompleted : "primary-replica resync finished but was not started";
                                if (state == IndexShardState.CLOSED) {
                                // ignore, shutting down
                                } else {
                                    failShard("exception during primary-replica resync", e);
                                }
                            }
                        });
                    } catch (final AlreadyClosedException e) {
                    // okay, the index was deleted
                    }
                }, null);
            }
        }
        // set this last, once we finished updating all internal state.
        this.shardRouting = newRouting;
        assert this.shardRouting.primary() == false || // note that we use started and not active to avoid relocating shards
        this.shardRouting.started() == false || // if permits are blocked, we are still transitioning
        this.indexShardOperationPermits.isBlocked() || this.replicationTracker.isPrimaryMode() : "a started primary with non-pending operation term must be in primary mode " + this.shardRouting;
        shardStateUpdated.countDown();
    }
    if (currentRouting.active() == false && newRouting.active()) {
        indexEventListener.afterIndexShardStarted(this);
    }
    if (newRouting.equals(currentRouting) == false) {
        indexEventListener.shardRoutingChanged(this, currentRouting, newRouting);
    }
    if (indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery == false) {
        final RetentionLeases retentionLeases = replicationTracker.getRetentionLeases();
        final Set<ShardRouting> shardRoutings = new HashSet<>(routingTable.getShards());
        // include relocation targets
        shardRoutings.addAll(routingTable.assignedShards());
        if (shardRoutings.stream().allMatch(shr -> shr.assignedToNode() && retentionLeases.contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(shr)))) {
            useRetentionLeasesInPeerRecovery = true;
            turnOffTranslogRetention();
        }
    }
}