Examples with NO_OPS_PERFORMED org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED used on opensource projects

Example 1 with NO_OPS_PERFORMED

use of org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED in project OpenSearch by opensearch-project.

the class ReplicationTrackerTests method testUpdateAllocationIdsFromMaster.

public void testUpdateAllocationIdsFromMaster() throws Exception {
    final long initialClusterStateVersion = randomNonNegativeLong();
    final int numberOfActiveAllocationsIds = randomIntBetween(2, 16);
    final int numberOfInitializingIds = randomIntBetween(2, 16);
    final Tuple<Set<AllocationId>, Set<AllocationId>> activeAndInitializingAllocationIds = randomActiveAndInitializingAllocationIds(numberOfActiveAllocationsIds, numberOfInitializingIds);
    final Set<AllocationId> activeAllocationIds = activeAndInitializingAllocationIds.v1();
    final Set<AllocationId> initializingIds = activeAndInitializingAllocationIds.v2();
    AllocationId primaryId = activeAllocationIds.iterator().next();
    IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId);
    final ReplicationTracker tracker = newTracker(primaryId);
    tracker.updateFromMaster(initialClusterStateVersion, ids(activeAllocationIds), routingTable);
    tracker.activatePrimaryMode(NO_OPS_PERFORMED);
    assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds)));
    assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable));
    // first we assert that the in-sync and tracking sets are set up correctly
    assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(activeAllocationIds.stream().filter(a -> a.equals(primaryId) == false).allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).getLocalCheckpoint() == SequenceNumbers.UNASSIGNED_SEQ_NO));
    assertTrue(initializingIds.stream().noneMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(initializingIds.stream().filter(a -> a.equals(primaryId) == false).allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).getLocalCheckpoint() == SequenceNumbers.UNASSIGNED_SEQ_NO));
    // now we will remove some allocation IDs from these and ensure that they propagate through
    final Set<AllocationId> removingActiveAllocationIds = new HashSet<>(randomSubsetOf(activeAllocationIds));
    removingActiveAllocationIds.remove(primaryId);
    final Set<AllocationId> newActiveAllocationIds = activeAllocationIds.stream().filter(a -> !removingActiveAllocationIds.contains(a)).collect(Collectors.toSet());
    final List<AllocationId> removingInitializingAllocationIds = randomSubsetOf(initializingIds);
    final Set<AllocationId> newInitializingAllocationIds = initializingIds.stream().filter(a -> !removingInitializingAllocationIds.contains(a)).collect(Collectors.toSet());
    routingTable = routingTable(newInitializingAllocationIds, primaryId);
    tracker.updateFromMaster(initialClusterStateVersion + 1, ids(newActiveAllocationIds), routingTable);
    assertTrue(newActiveAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(removingActiveAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()) == null));
    assertTrue(newInitializingAllocationIds.stream().noneMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(removingInitializingAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()) == null));
    assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(Sets.difference(Sets.union(activeAllocationIds, newActiveAllocationIds), removingActiveAllocationIds))));
    assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable));
    /*
         * Now we will add an allocation ID to each of active and initializing and ensure they propagate through. Using different lengths
         * than we have been using above ensures that we can not collide with a previous allocation ID
         */
    newInitializingAllocationIds.add(AllocationId.newInitializing());
    tracker.updateFromMaster(initialClusterStateVersion + 2, ids(newActiveAllocationIds), routingTable(newInitializingAllocationIds, primaryId));
    assertTrue(newActiveAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(newActiveAllocationIds.stream().filter(a -> a.equals(primaryId) == false).allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).getLocalCheckpoint() == SequenceNumbers.UNASSIGNED_SEQ_NO));
    assertTrue(newInitializingAllocationIds.stream().noneMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
    assertTrue(newInitializingAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).getLocalCheckpoint() == SequenceNumbers.UNASSIGNED_SEQ_NO));
    // the tracking allocation IDs should play no role in determining the global checkpoint
    final Map<AllocationId, Integer> activeLocalCheckpoints = newActiveAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024)));
    activeLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l));
    final Map<AllocationId, Integer> initializingLocalCheckpoints = newInitializingAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024)));
    initializingLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l));
    assertTrue(activeLocalCheckpoints.entrySet().stream().allMatch(e -> tracker.getTrackedLocalCheckpointForShard(e.getKey().getId()).getLocalCheckpoint() == e.getValue()));
    assertTrue(initializingLocalCheckpoints.entrySet().stream().allMatch(e -> tracker.getTrackedLocalCheckpointForShard(e.getKey().getId()).getLocalCheckpoint() == e.getValue()));
    final long minimumActiveLocalCheckpoint = (long) activeLocalCheckpoints.values().stream().min(Integer::compareTo).get();
    assertThat(tracker.getGlobalCheckpoint(), equalTo(minimumActiveLocalCheckpoint));
    assertThat(updatedGlobalCheckpoint.get(), equalTo(minimumActiveLocalCheckpoint));
    final long minimumInitailizingLocalCheckpoint = (long) initializingLocalCheckpoints.values().stream().min(Integer::compareTo).get();
    // now we are going to add a new allocation ID and bring it in sync which should move it to the in-sync allocation IDs
    final long localCheckpoint = randomIntBetween(0, Math.toIntExact(Math.min(minimumActiveLocalCheckpoint, minimumInitailizingLocalCheckpoint) - 1));
    // using a different length than we have been using above ensures that we can not collide with a previous allocation ID
    final AllocationId newSyncingAllocationId = AllocationId.newInitializing();
    newInitializingAllocationIds.add(newSyncingAllocationId);
    tracker.updateFromMaster(initialClusterStateVersion + 3, ids(newActiveAllocationIds), routingTable(newInitializingAllocationIds, primaryId));
    addPeerRecoveryRetentionLease(tracker, newSyncingAllocationId);
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final Thread thread = new Thread(() -> {
        try {
            barrier.await();
            tracker.initiateTracking(newSyncingAllocationId.getId());
            tracker.markAllocationIdAsInSync(newSyncingAllocationId.getId(), localCheckpoint);
            barrier.await();
        } catch (final BrokenBarrierException | InterruptedException e) {
            throw new RuntimeException(e);
        }
    });
    thread.start();
    barrier.await();
    assertBusy(() -> {
        assertTrue(tracker.pendingInSync.contains(newSyncingAllocationId.getId()));
        assertFalse(tracker.getTrackedLocalCheckpointForShard(newSyncingAllocationId.getId()).inSync);
    });
    tracker.updateLocalCheckpoint(newSyncingAllocationId.getId(), randomIntBetween(Math.toIntExact(minimumActiveLocalCheckpoint), 1024));
    barrier.await();
    assertFalse(tracker.pendingInSync.contains(newSyncingAllocationId.getId()));
    assertTrue(tracker.getTrackedLocalCheckpointForShard(newSyncingAllocationId.getId()).inSync);
    /*
         * The new in-sync allocation ID is in the in-sync set now yet the master does not know this; the allocation ID should still be in
         * the in-sync set even if we receive a cluster state update that does not reflect this.
         *
         */
    tracker.updateFromMaster(initialClusterStateVersion + 4, ids(newActiveAllocationIds), routingTable(newInitializingAllocationIds, primaryId));
    assertTrue(tracker.getTrackedLocalCheckpointForShard(newSyncingAllocationId.getId()).inSync);
    assertFalse(tracker.pendingInSync.contains(newSyncingAllocationId.getId()));
}

Example 2 with NO_OPS_PERFORMED

use of org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED in project OpenSearch by opensearch-project.

the class ReplicationTrackerTests method testPeerRecoveryRetentionLeaseCreationAndRenewal.

public void testPeerRecoveryRetentionLeaseCreationAndRenewal() {
    final int numberOfActiveAllocationsIds = randomIntBetween(1, 8);
    final int numberOfInitializingIds = randomIntBetween(0, 8);
    final Tuple<Set<AllocationId>, Set<AllocationId>> activeAndInitializingAllocationIds = randomActiveAndInitializingAllocationIds(numberOfActiveAllocationsIds, numberOfInitializingIds);
    final Set<AllocationId> activeAllocationIds = activeAndInitializingAllocationIds.v1();
    final Set<AllocationId> initializingAllocationIds = activeAndInitializingAllocationIds.v2();
    final AllocationId primaryId = activeAllocationIds.iterator().next();
    final long initialClusterStateVersion = randomNonNegativeLong();
    final AtomicLong currentTimeMillis = new AtomicLong(0L);
    final ReplicationTracker tracker = newTracker(primaryId, updatedGlobalCheckpoint::set, currentTimeMillis::get);
    final long retentionLeaseExpiryTimeMillis = tracker.indexSettings().getRetentionLeaseMillis();
    final long peerRecoveryRetentionLeaseRenewalTimeMillis = retentionLeaseExpiryTimeMillis / 2;
    final long maximumTestTimeMillis = 13 * retentionLeaseExpiryTimeMillis;
    final long testStartTimeMillis = randomLongBetween(0L, Long.MAX_VALUE - maximumTestTimeMillis);
    currentTimeMillis.set(testStartTimeMillis);
    final Function<AllocationId, RetentionLease> retentionLeaseFromAllocationId = allocationId -> new RetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(nodeIdFromAllocationId(allocationId)), 0L, currentTimeMillis.get(), ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE);
    final List<RetentionLease> initialLeases = new ArrayList<>();
    if (randomBoolean()) {
        initialLeases.add(retentionLeaseFromAllocationId.apply(primaryId));
    }
    for (final AllocationId replicaId : initializingAllocationIds) {
        if (randomBoolean()) {
            initialLeases.add(retentionLeaseFromAllocationId.apply(replicaId));
        }
    }
    for (int i = randomIntBetween(0, 5); i > 0; i--) {
        initialLeases.add(retentionLeaseFromAllocationId.apply(AllocationId.newInitializing()));
    }
    tracker.updateRetentionLeasesOnReplica(new RetentionLeases(randomNonNegativeLong(), randomNonNegativeLong(), initialLeases));
    IndexShardRoutingTable routingTable = routingTable(initializingAllocationIds, primaryId);
    tracker.updateFromMaster(initialClusterStateVersion, ids(activeAllocationIds), routingTable);
    tracker.activatePrimaryMode(NO_OPS_PERFORMED);
    assertTrue("primary's retention lease should exist", tracker.getRetentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(routingTable.primaryShard())));
    final Consumer<Runnable> assertAsTimePasses = assertion -> {
        final long startTime = currentTimeMillis.get();
        while (currentTimeMillis.get() < startTime + retentionLeaseExpiryTimeMillis * 2) {
            currentTimeMillis.addAndGet(randomLongBetween(0L, retentionLeaseExpiryTimeMillis * 2));
            tracker.renewPeerRecoveryRetentionLeases();
            tracker.getRetentionLeases(true);
            assertion.run();
        }
    };
    assertAsTimePasses.accept(() -> {
        // Leases for assigned replicas do not expire
        final RetentionLeases retentionLeases = tracker.getRetentionLeases();
        for (final AllocationId replicaId : initializingAllocationIds) {
            final String leaseId = retentionLeaseFromAllocationId.apply(replicaId).id();
            assertTrue("should not have removed lease for " + replicaId + " in " + retentionLeases, initialLeases.stream().noneMatch(l -> l.id().equals(leaseId)) || retentionLeases.contains(leaseId));
        }
    });
    // Leases that don't correspond to assigned replicas, however, are expired by this time.
    final Set<String> expectedLeaseIds = Stream.concat(Stream.of(primaryId), initializingAllocationIds.stream()).map(allocationId -> retentionLeaseFromAllocationId.apply(allocationId).id()).collect(Collectors.toSet());
    for (final RetentionLease retentionLease : tracker.getRetentionLeases().leases()) {
        assertThat(expectedLeaseIds, hasItem(retentionLease.id()));
    }
    for (AllocationId replicaId : initializingAllocationIds) {
        markAsTrackingAndInSyncQuietly(tracker, replicaId.getId(), NO_OPS_PERFORMED);
    }
    assertThat(tracker.getRetentionLeases().leases().stream().map(RetentionLease::id).collect(Collectors.toSet()), equalTo(expectedLeaseIds));
    assertAsTimePasses.accept(() -> {
        // Leases still don't expire
        assertThat(tracker.getRetentionLeases().leases().stream().map(RetentionLease::id).collect(Collectors.toSet()), equalTo(expectedLeaseIds));
        // Also leases are renewed before reaching half the expiry time
        // noinspection OptionalGetWithoutIsPresent
        assertThat(tracker.getRetentionLeases() + " renewed before too long", tracker.getRetentionLeases().leases().stream().mapToLong(RetentionLease::timestamp).min().getAsLong(), greaterThanOrEqualTo(currentTimeMillis.get() - peerRecoveryRetentionLeaseRenewalTimeMillis));
    });
    IndexShardRoutingTable.Builder routingTableBuilder = new IndexShardRoutingTable.Builder(routingTable);
    for (ShardRouting replicaShard : routingTable.replicaShards()) {
        routingTableBuilder.removeShard(replicaShard);
        routingTableBuilder.addShard(replicaShard.moveToStarted());
    }
    routingTable = routingTableBuilder.build();
    activeAllocationIds.addAll(initializingAllocationIds);
    tracker.updateFromMaster(initialClusterStateVersion + randomLongBetween(1, 10), ids(activeAllocationIds), routingTable);
    assertAsTimePasses.accept(() -> {
        // Leases still don't expire
        assertThat(tracker.getRetentionLeases().leases().stream().map(RetentionLease::id).collect(Collectors.toSet()), equalTo(expectedLeaseIds));
        // ... and any extra peer recovery retention leases are expired immediately since the shard is fully active
        tracker.addPeerRecoveryRetentionLease(randomAlphaOfLength(10), randomNonNegativeLong(), ActionListener.wrap(() -> {
        }));
    });
    tracker.renewPeerRecoveryRetentionLeases();
    assertTrue("expired extra lease", tracker.getRetentionLeases(true).v1());
    final AllocationId advancingAllocationId = initializingAllocationIds.isEmpty() || rarely() ? primaryId : randomFrom(initializingAllocationIds);
    final String advancingLeaseId = retentionLeaseFromAllocationId.apply(advancingAllocationId).id();
    final long initialGlobalCheckpoint = Math.max(NO_OPS_PERFORMED, tracker.getTrackedLocalCheckpointForShard(advancingAllocationId.getId()).globalCheckpoint);
    assertThat(tracker.getRetentionLeases().get(advancingLeaseId).retainingSequenceNumber(), equalTo(initialGlobalCheckpoint + 1));
    final long newGlobalCheckpoint = initialGlobalCheckpoint + randomLongBetween(1, 1000);
    tracker.updateGlobalCheckpointForShard(advancingAllocationId.getId(), newGlobalCheckpoint);
    tracker.renewPeerRecoveryRetentionLeases();
    assertThat("lease was renewed because the shard advanced its global checkpoint", tracker.getRetentionLeases().get(advancingLeaseId).retainingSequenceNumber(), equalTo(newGlobalCheckpoint + 1));
    final long initialVersion = tracker.getRetentionLeases().version();
    tracker.renewPeerRecoveryRetentionLeases();
    assertThat("immediate renewal is a no-op", tracker.getRetentionLeases().version(), equalTo(initialVersion));
    // noinspection OptionalGetWithoutIsPresent
    final long millisUntilFirstRenewal = tracker.getRetentionLeases().leases().stream().mapToLong(RetentionLease::timestamp).min().getAsLong() + peerRecoveryRetentionLeaseRenewalTimeMillis - currentTimeMillis.get();
    if (millisUntilFirstRenewal != 0) {
        final long shorterThanRenewalTime = randomLongBetween(0L, millisUntilFirstRenewal - 1);
        currentTimeMillis.addAndGet(shorterThanRenewalTime);
        tracker.renewPeerRecoveryRetentionLeases();
        assertThat("renewal is a no-op after a short time", tracker.getRetentionLeases().version(), equalTo(initialVersion));
        currentTimeMillis.addAndGet(millisUntilFirstRenewal - shorterThanRenewalTime);
    }
    tracker.renewPeerRecoveryRetentionLeases();
    assertThat("renewal happens after a sufficiently long time", tracker.getRetentionLeases().version(), greaterThan(initialVersion));
    assertTrue("all leases were renewed", tracker.getRetentionLeases().leases().stream().allMatch(l -> l.timestamp() == currentTimeMillis.get()));
    assertThat("test ran for too long, potentially leading to overflow", currentTimeMillis.get(), lessThanOrEqualTo(testStartTimeMillis + maximumTestTimeMillis));
}

Example 3 with NO_OPS_PERFORMED

use of org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED in project OpenSearch by opensearch-project.

the class GlobalCheckpointListenersTests method testFailingListenerAfterTimeout.

public void testFailingListenerAfterTimeout() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final Logger mockLogger = mock(Logger.class);
    doAnswer(invocationOnMock -> {
        latch.countDown();
        return null;
    }).when(mockLogger).warn(any(String.class), any(RuntimeException.class));
    final GlobalCheckpointListeners globalCheckpointListeners = new GlobalCheckpointListeners(shardId, scheduler, mockLogger);
    final TimeValue timeout = TimeValue.timeValueMillis(randomIntBetween(1, 50));
    globalCheckpointListeners.add(NO_OPS_PERFORMED, maybeMultipleInvocationProtectingListener((g, e) -> {
        throw new RuntimeException("failure");
    }), timeout);
    latch.await();
    final ArgumentCaptor<String> message = ArgumentCaptor.forClass(String.class);
    final ArgumentCaptor<RuntimeException> t = ArgumentCaptor.forClass(RuntimeException.class);
    verify(mockLogger).warn(message.capture(), t.capture());
    assertThat(message.getValue(), equalTo("error notifying global checkpoint listener of timeout"));
    assertNotNull(t.getValue());
    assertThat(t.getValue(), instanceOf(RuntimeException.class));
    assertThat(t.getValue().getMessage(), equalTo("failure"));
}

Example 4 with NO_OPS_PERFORMED

use of org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED in project OpenSearch by opensearch-project.

the class GlobalCheckpointListenersTests method testFailingListenerReadyToBeNotified.

public void testFailingListenerReadyToBeNotified() {
    final Logger mockLogger = mock(Logger.class);
    final GlobalCheckpointListeners globalCheckpointListeners = new GlobalCheckpointListeners(shardId, scheduler, mockLogger);
    final long globalCheckpoint = randomLongBetween(NO_OPS_PERFORMED + 1, Long.MAX_VALUE);
    globalCheckpointListeners.globalCheckpointUpdated(globalCheckpoint);
    final int numberOfListeners = randomIntBetween(0, 16);
    final long[] globalCheckpoints = new long[numberOfListeners];
    for (int i = 0; i < numberOfListeners; i++) {
        final int index = i;
        final boolean failure = randomBoolean();
        globalCheckpointListeners.add(randomLongBetween(NO_OPS_PERFORMED, globalCheckpoint - 1), maybeMultipleInvocationProtectingListener((g, e) -> {
            if (failure) {
                globalCheckpoints[index] = Long.MIN_VALUE;
                throw new RuntimeException("failure");
            } else {
                globalCheckpoints[index] = globalCheckpoint;
            }
        }), null);
        // the listener should be notified immediately
        if (failure) {
            assertThat(globalCheckpoints[i], equalTo(Long.MIN_VALUE));
            final ArgumentCaptor<ParameterizedMessage> message = ArgumentCaptor.forClass(ParameterizedMessage.class);
            final ArgumentCaptor<RuntimeException> t = ArgumentCaptor.forClass(RuntimeException.class);
            verify(mockLogger).warn(message.capture(), t.capture());
            reset(mockLogger);
            assertThat(message.getValue().getFormat(), equalTo("error notifying global checkpoint listener of updated global checkpoint [{}]"));
            assertNotNull(message.getValue().getParameters());
            assertThat(message.getValue().getParameters().length, equalTo(1));
            assertThat(message.getValue().getParameters()[0], equalTo(globalCheckpoint));
            assertNotNull(t.getValue());
            assertThat(t.getValue().getMessage(), equalTo("failure"));
        } else {
            assertThat(globalCheckpoints[i], equalTo(globalCheckpoint));
        }
    }
}

Example 5 with NO_OPS_PERFORMED

use of org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED in project OpenSearch by opensearch-project.

the class GlobalCheckpointListenersTests method testNotificationOnClosedUsesExecutor.

public void testNotificationOnClosedUsesExecutor() throws IOException {
    final AtomicInteger count = new AtomicInteger();
    final Executor executor = command -> {
        count.incrementAndGet();
        command.run();
    };
    final GlobalCheckpointListeners globalCheckpointListeners = new GlobalCheckpointListeners(shardId, scheduler, logger);
    globalCheckpointListeners.close();
    final AtomicInteger notified = new AtomicInteger();
    final int numberOfListeners = randomIntBetween(0, 16);
    for (int i = 0; i < numberOfListeners; i++) {
        globalCheckpointListeners.add(NO_OPS_PERFORMED, maybeMultipleInvocationProtectingListener(new TestGlobalCheckpointListener() {

            @Override
            public Executor executor() {
                return executor;
            }

            @Override
            public void accept(final long g, final Exception e) {
                notified.incrementAndGet();
                assertThat(g, equalTo(UNASSIGNED_SEQ_NO));
                assertNotNull(e);
                assertThat(e, instanceOf(IndexShardClosedException.class));
                assertThat(((IndexShardClosedException) e).getShardId(), equalTo(shardId));
            }
        }), null);
    }
    assertThat(notified.get(), equalTo(numberOfListeners));
    assertThat(count.get(), equalTo(numberOfListeners));
}