Search in sources :

Example 1 with Priority

use of org.opensearch.common.Priority in project OpenSearch by opensearch-project.

the class SnapshotsService method createDeleteStateUpdate.

private ClusterStateUpdateTask createDeleteStateUpdate(List<SnapshotId> snapshotIds, String repoName, RepositoryData repositoryData, Priority priority, ActionListener<Void> listener) {
    // Short circuit to noop state update if there isn't anything to delete
    if (snapshotIds.isEmpty()) {
        return new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                listener.onFailure(e);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                listener.onResponse(null);
            }
        };
    }
    return new ClusterStateUpdateTask(priority) {

        private SnapshotDeletionsInProgress.Entry newDelete;

        private boolean reusedExistingDelete = false;

        // Snapshots that had all of their shard snapshots in queued state and thus were removed from the
        // cluster state right away
        private final Collection<Snapshot> completedNoCleanup = new ArrayList<>();

        // Snapshots that were aborted and that already wrote data to the repository and now have to be deleted
        // from the repository after the cluster state update
        private final Collection<SnapshotsInProgress.Entry> completedWithCleanup = new ArrayList<>();

        @Override
        public ClusterState execute(ClusterState currentState) {
            final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
            final Version minNodeVersion = currentState.nodes().getMinNodeVersion();
            if (minNodeVersion.before(FULL_CONCURRENCY_VERSION)) {
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete - another snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
            }
            final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
            if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete snapshots while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]");
            }
            final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY);
            for (RestoreInProgress.Entry entry : restoreInProgress) {
                if (repoName.equals(entry.snapshot().getRepository()) && snapshotIds.contains(entry.snapshot().getSnapshotId())) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]");
                }
            }
            final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final Set<SnapshotId> activeCloneSources = snapshots.entries().stream().filter(SnapshotsInProgress.Entry::isClone).map(SnapshotsInProgress.Entry::source).collect(Collectors.toSet());
            for (SnapshotId snapshotId : snapshotIds) {
                if (activeCloneSources.contains(snapshotId)) {
                    throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotId), "cannot delete snapshot while it is being cloned");
                }
            }
            // Snapshot ids that will have to be physically deleted from the repository
            final Set<SnapshotId> snapshotIdsRequiringCleanup = new HashSet<>(snapshotIds);
            final SnapshotsInProgress updatedSnapshots;
            if (minNodeVersion.onOrAfter(FULL_CONCURRENCY_VERSION)) {
                updatedSnapshots = SnapshotsInProgress.of(snapshots.entries().stream().map(existing -> {
                    if (existing.state() == State.STARTED && snapshotIdsRequiringCleanup.contains(existing.snapshot().getSnapshotId())) {
                        // snapshot is started - mark every non completed shard as aborted
                        final SnapshotsInProgress.Entry abortedEntry = existing.abort();
                        if (abortedEntry == null) {
                            // No work has been done for this snapshot yet so we remove it from the cluster state directly
                            final Snapshot existingNotYetStartedSnapshot = existing.snapshot();
                            // any leaked listener assertions
                            if (endingSnapshots.add(existingNotYetStartedSnapshot)) {
                                completedNoCleanup.add(existingNotYetStartedSnapshot);
                            }
                            snapshotIdsRequiringCleanup.remove(existingNotYetStartedSnapshot.getSnapshotId());
                        } else if (abortedEntry.state().completed()) {
                            completedWithCleanup.add(abortedEntry);
                        }
                        return abortedEntry;
                    }
                    return existing;
                }).filter(Objects::nonNull).collect(Collectors.toList()));
                if (snapshotIdsRequiringCleanup.isEmpty()) {
                    // We only saw snapshots that could be removed from the cluster state right away, no need to update the deletions
                    return updateWithSnapshots(currentState, updatedSnapshots, null);
                }
            } else {
                if (snapshots.entries().isEmpty() == false) {
                    // However other snapshots are running - cannot continue
                    throw new ConcurrentSnapshotExecutionException(repoName, snapshotIds.toString(), "another snapshot is currently running cannot delete");
                }
                updatedSnapshots = snapshots;
            }
            // add the snapshot deletion to the cluster state
            final SnapshotDeletionsInProgress.Entry replacedEntry = deletionsInProgress.getEntries().stream().filter(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.WAITING).findFirst().orElse(null);
            if (replacedEntry == null) {
                final Optional<SnapshotDeletionsInProgress.Entry> foundDuplicate = deletionsInProgress.getEntries().stream().filter(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED && entry.getSnapshots().containsAll(snapshotIds)).findFirst();
                if (foundDuplicate.isPresent()) {
                    newDelete = foundDuplicate.get();
                    reusedExistingDelete = true;
                    return currentState;
                }
                final List<SnapshotId> toDelete = Collections.unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup));
                ensureBelowConcurrencyLimit(repoName, toDelete.get(0).getName(), snapshots, deletionsInProgress);
                newDelete = new SnapshotDeletionsInProgress.Entry(toDelete, repoName, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), updatedSnapshots.entries().stream().filter(entry -> repoName.equals(entry.repository())).noneMatch(SnapshotsService::isWritingToRepository) && deletionsInProgress.getEntries().stream().noneMatch(entry -> repoName.equals(entry.repository()) && entry.state() == SnapshotDeletionsInProgress.State.STARTED) ? SnapshotDeletionsInProgress.State.STARTED : SnapshotDeletionsInProgress.State.WAITING);
            } else {
                newDelete = replacedEntry.withAddedSnapshots(snapshotIdsRequiringCleanup);
            }
            return updateWithSnapshots(currentState, updatedSnapshots, (replacedEntry == null ? deletionsInProgress : deletionsInProgress.withRemovedEntry(replacedEntry.uuid())).withAddedEntry(newDelete));
        }

        @Override
        public void onFailure(String source, Exception e) {
            endingSnapshots.removeAll(completedNoCleanup);
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (completedNoCleanup.isEmpty() == false) {
                logger.info("snapshots {} aborted", completedNoCleanup);
            }
            for (Snapshot snapshot : completedNoCleanup) {
                failSnapshotCompletionListeners(snapshot, new SnapshotException(snapshot, SnapshotsInProgress.ABORTED_FAILURE_TEXT));
            }
            if (newDelete == null) {
                listener.onResponse(null);
            } else {
                addDeleteListener(newDelete.uuid(), listener);
                if (reusedExistingDelete) {
                    return;
                }
                if (newDelete.state() == SnapshotDeletionsInProgress.State.STARTED) {
                    if (tryEnterRepoLoop(repoName)) {
                        deleteSnapshotsFromRepository(newDelete, repositoryData, newState.nodes().getMinNodeVersion());
                    } else {
                        logger.trace("Delete [{}] could not execute directly and was queued", newDelete);
                    }
                } else {
                    for (SnapshotsInProgress.Entry completedSnapshot : completedWithCleanup) {
                        endSnapshot(completedSnapshot, newState.metadata(), repositoryData);
                    }
                }
            }
        }
    };
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) Collections.unmodifiableList(java.util.Collections.unmodifiableList) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Regex(org.opensearch.common.regex.Regex) Strings(org.opensearch.common.Strings) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) ActionFilters(org.opensearch.action.support.ActionFilters) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) StepListener(org.opensearch.action.StepListener) State(org.opensearch.cluster.SnapshotsInProgress.State) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) RepositoriesService(org.opensearch.repositories.RepositoriesService) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) IOException(java.io.IOException) DeleteSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest) ClusterService(org.opensearch.cluster.service.ClusterService) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RoutingTable(org.opensearch.cluster.routing.RoutingTable) SnapshotsInProgress.completed(org.opensearch.cluster.SnapshotsInProgress.completed) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ShardGenerations(org.opensearch.repositories.ShardGenerations) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) Locale(java.util.Locale) NotMasterException(org.opensearch.cluster.NotMasterException) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) List(java.util.List) Optional(java.util.Optional) ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) CloneSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest) HashMap(java.util.HashMap) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) UUIDs(org.opensearch.common.UUIDs) LinkedList(java.util.LinkedList) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Version(org.opensearch.Version) LegacyESVersion(org.opensearch.LegacyESVersion) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) IOException(java.io.IOException) NotMasterException(org.opensearch.cluster.NotMasterException) RepositoryException(org.opensearch.repositories.RepositoryException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) Collection(java.util.Collection) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress)

Example 2 with Priority

use of org.opensearch.common.Priority in project OpenSearch by opensearch-project.

the class DiskThresholdMonitorTests method testAutoReleaseIndices.

public void testAutoReleaseIndices() {
    AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
    AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
    AllocationService allocation = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1)).put(IndexMetadata.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test_1")).addAsNew(metadata.index("test_2")).build();
    final ClusterState clusterState = applyStartedShardsUntilNoChange(ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build(), allocation);
    assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8));
    final ImmutableOpenMap.Builder<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpacesBuilder = ImmutableOpenMap.builder();
    final int reservedSpaceNode1 = between(0, 10);
    reservedSpacesBuilder.put(new ClusterInfo.NodeAndPath("node1", "/foo/bar"), new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode1).build());
    final int reservedSpaceNode2 = between(0, 10);
    reservedSpacesBuilder.put(new ClusterInfo.NodeAndPath("node2", "/foo/bar"), new ClusterInfo.ReservedSpace.Builder().add(new ShardId("", "", 0), reservedSpaceNode2).build());
    ImmutableOpenMap<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpaces = reservedSpacesBuilder.build();
    DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterState, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () -> 0L, (reason, priority, listener) -> {
        assertNotNull(listener);
        assertThat(priority, equalTo(Priority.HIGH));
        listener.onResponse(clusterState);
    }) {

        @Override
        protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
            if (readOnly) {
                assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
            } else {
                assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
            }
            listener.onResponse(null);
        }
    };
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
    builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
    monitor.onNewInfo(clusterInfo(builder.build(), reservedSpaces));
    assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
    assertNull(indicesToRelease.get());
    // Reserved space is ignored when applying block
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 90)));
    builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 90)));
    monitor.onNewInfo(clusterInfo(builder.build(), reservedSpaces));
    assertNull(indicesToMarkReadOnly.get());
    assertNull(indicesToRelease.get());
    // Change cluster state so that "test_2" index is blocked (read only)
    IndexMetadata indexMetadata = IndexMetadata.builder(clusterState.metadata().index("test_2")).settings(Settings.builder().put(clusterState.metadata().index("test_2").getSettings()).put(IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
    ClusterState clusterStateWithBlocks = ClusterState.builder(clusterState).metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build()).blocks(ClusterBlocks.builder().addBlocks(indexMetadata).build()).build();
    assertTrue(clusterStateWithBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_2"));
    monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithBlocks, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () -> 0L, (reason, priority, listener) -> {
        assertNotNull(listener);
        assertThat(priority, equalTo(Priority.HIGH));
        listener.onResponse(clusterStateWithBlocks);
    }) {

        @Override
        protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
            if (readOnly) {
                assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
            } else {
                assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
            }
            listener.onResponse(null);
        }
    };
    // When free disk on any of node1 or node2 goes below 5% flood watermark, then apply index block on indices not having the block
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 100)));
    builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
    monitor.onNewInfo(clusterInfo(builder.build(), reservedSpaces));
    assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
    assertNull(indicesToRelease.get());
    // When free disk on node1 and node2 goes above 10% high watermark then release index block, ignoring reserved space
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
    builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
    monitor.onNewInfo(clusterInfo(builder.build(), reservedSpaces));
    assertNull(indicesToMarkReadOnly.get());
    assertThat(indicesToRelease.get(), contains("test_2"));
    // When no usage information is present for node2, we don't release the block
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
    monitor.onNewInfo(clusterInfo(builder.build()));
    assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
    assertNull(indicesToRelease.get());
    // When disk usage on one node is between the high and flood-stage watermarks, nothing changes
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
    builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 100)));
    if (randomBoolean()) {
        builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
    }
    monitor.onNewInfo(clusterInfo(builder.build()));
    assertNull(indicesToMarkReadOnly.get());
    assertNull(indicesToRelease.get());
    // When disk usage on one node is missing and the other is below the high watermark, nothing changes
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 100)));
    if (randomBoolean()) {
        builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
    }
    monitor.onNewInfo(clusterInfo(builder.build()));
    assertNull(indicesToMarkReadOnly.get());
    assertNull(indicesToRelease.get());
    // When disk usage on one node is missing and the other is above the flood-stage watermark, affected indices are blocked
    indicesToMarkReadOnly.set(null);
    indicesToRelease.set(null);
    builder = ImmutableOpenMap.builder();
    builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
    if (randomBoolean()) {
        builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
    }
    monitor.onNewInfo(clusterInfo(builder.build()));
    assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
    assertNull(indicesToRelease.get());
}
Also used : DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) MockLogAppender(org.opensearch.test.MockLogAppender) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) LongSupplier(java.util.function.LongSupplier) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) Level(org.apache.logging.log4j.Level) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Version(org.opensearch.Version) Priority(org.opensearch.common.Priority) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) OpenSearchAllocationTestCase(org.opensearch.cluster.OpenSearchAllocationTestCase) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ActionListener(org.opensearch.action.ActionListener) DiskUsage(org.opensearch.cluster.DiskUsage) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) ClusterInfo(org.opensearch.cluster.ClusterInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ShardId(org.opensearch.index.shard.ShardId) TestLogging(org.opensearch.test.junit.annotations.TestLogging) AtomicLong(java.util.concurrent.atomic.AtomicLong) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ClusterName(org.opensearch.cluster.ClusterName) RoutingTable(org.opensearch.cluster.routing.RoutingTable) RoutingNode(org.opensearch.cluster.routing.RoutingNode) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Metadata(org.opensearch.cluster.metadata.Metadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) DiskUsage(org.opensearch.cluster.DiskUsage) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ShardId(org.opensearch.index.shard.ShardId) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ClusterState(org.opensearch.cluster.ClusterState) AtomicReference(java.util.concurrent.atomic.AtomicReference) ClusterInfo(org.opensearch.cluster.ClusterInfo) RoutingTable(org.opensearch.cluster.routing.RoutingTable) ActionListener(org.opensearch.action.ActionListener)

Example 3 with Priority

use of org.opensearch.common.Priority in project OpenSearch by opensearch-project.

the class BatchedRerouteServiceTests method testBatchesReroutesTogetherAtPriorityOfHighestSubmittedReroute.

public void testBatchesReroutesTogetherAtPriorityOfHighestSubmittedReroute() throws BrokenBarrierException, InterruptedException {
    final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
    clusterService.submitStateUpdateTask("block master service", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            // notify test that we are blocked
            cyclicBarrier.await();
            // wait to be unblocked by test
            cyclicBarrier.await();
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            throw new AssertionError(source, e);
        }
    });
    // wait for master thread to be blocked
    cyclicBarrier.await();
    final AtomicBoolean rerouteExecuted = new AtomicBoolean();
    final BatchedRerouteService batchedRerouteService = new BatchedRerouteService(clusterService, (s, r) -> {
        // only called once
        assertTrue(rerouteExecuted.compareAndSet(false, true));
        return s;
    });
    final int iterations = scaledRandomIntBetween(1, 100);
    final CountDownLatch tasksSubmittedCountDown = new CountDownLatch(iterations);
    final CountDownLatch tasksCompletedCountDown = new CountDownLatch(iterations);
    final List<Runnable> actions = new ArrayList<>(iterations);
    final Function<Priority, Runnable> rerouteFromPriority = priority -> () -> {
        final AtomicBoolean alreadyRun = new AtomicBoolean();
        batchedRerouteService.reroute("reroute at " + priority, priority, ActionListener.wrap(() -> {
            assertTrue(alreadyRun.compareAndSet(false, true));
            tasksCompletedCountDown.countDown();
        }));
        tasksSubmittedCountDown.countDown();
    };
    // ensure at least one URGENT priority reroute
    actions.add(rerouteFromPriority.apply(Priority.URGENT));
    for (int i = 1; i < iterations; i++) {
        final int iteration = i;
        if (randomBoolean()) {
            actions.add(rerouteFromPriority.apply(randomFrom(Priority.LOW, Priority.NORMAL, Priority.HIGH, Priority.URGENT)));
        } else {
            final Priority priority = randomFrom(Priority.NORMAL, Priority.HIGH, Priority.URGENT, Priority.IMMEDIATE);
            final boolean submittedConcurrentlyWithReroute = randomBoolean();
            if (submittedConcurrentlyWithReroute == false) {
                // this task might be submitted later
                tasksSubmittedCountDown.countDown();
            }
            actions.add(() -> {
                clusterService.submitStateUpdateTask("other task " + iteration + " at " + priority, new ClusterStateUpdateTask(priority) {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        switch(priority) {
                            case IMMEDIATE:
                                if (submittedConcurrentlyWithReroute) {
                                    assertFalse("should have rerouted after " + priority + " priority task", rerouteExecuted.get());
                                }
                                // else this task might be submitted too late to precede the reroute
                                break;
                            case URGENT:
                                // may run either before or after reroute
                                break;
                            case HIGH:
                            case NORMAL:
                                assertTrue("should have rerouted before " + priority + " priority task", rerouteExecuted.get());
                                break;
                            default:
                                fail("unexpected priority: " + priority);
                                break;
                        }
                        return currentState;
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        throw new AssertionError(source, e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        tasksCompletedCountDown.countDown();
                    }
                });
                if (submittedConcurrentlyWithReroute) {
                    tasksSubmittedCountDown.countDown();
                }
            });
        }
    }
    Randomness.shuffle(actions);
    actions.forEach(threadPool.generic()::execute);
    assertTrue(tasksSubmittedCountDown.await(10, TimeUnit.SECONDS));
    // allow master thread to continue;
    cyclicBarrier.await();
    // wait for reroute to complete
    assertTrue(tasksCompletedCountDown.await(10, TimeUnit.SECONDS));
    // see above for assertion that it's only called once
    assertTrue(rerouteExecuted.get());
}
Also used : DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) ThreadPool(org.opensearch.threadpool.ThreadPool) TestThreadPool(org.opensearch.threadpool.TestThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Priority(org.opensearch.common.Priority) OpenSearchException(org.opensearch.OpenSearchException) Function(java.util.function.Function) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) After(org.junit.After) Matchers.lessThan(org.hamcrest.Matchers.lessThan) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Before(org.junit.Before) CyclicBarrier(java.util.concurrent.CyclicBarrier) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) Randomness(org.opensearch.common.Randomness) ClusterService(org.opensearch.cluster.service.ClusterService) ClusterServiceUtils(org.opensearch.test.ClusterServiceUtils) ClusterState(org.opensearch.cluster.ClusterState) Priority(org.opensearch.common.Priority) ArrayList(java.util.ArrayList) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) CountDownLatch(java.util.concurrent.CountDownLatch) OpenSearchException(org.opensearch.OpenSearchException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) CyclicBarrier(java.util.concurrent.CyclicBarrier) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean)

Example 4 with Priority

use of org.opensearch.common.Priority in project OpenSearch by opensearch-project.

the class RestClusterHealthActionTests method testFromRequest.

public void testFromRequest() {
    Map<String, String> params = new HashMap<>();
    String index = "index";
    boolean local = randomBoolean();
    String masterTimeout = randomTimeValue();
    String timeout = randomTimeValue();
    ClusterHealthStatus waitForStatus = randomFrom(ClusterHealthStatus.values());
    boolean waitForNoRelocatingShards = randomBoolean();
    boolean waitForNoInitializingShards = randomBoolean();
    int waitForActiveShards = randomIntBetween(1, 3);
    String waitForNodes = "node";
    Priority waitForEvents = randomFrom(Priority.values());
    params.put("index", index);
    params.put("local", String.valueOf(local));
    params.put("master_timeout", masterTimeout);
    params.put("timeout", timeout);
    params.put("wait_for_status", waitForStatus.name());
    if (waitForNoRelocatingShards || randomBoolean()) {
        params.put("wait_for_no_relocating_shards", String.valueOf(waitForNoRelocatingShards));
    }
    if (waitForNoInitializingShards || randomBoolean()) {
        params.put("wait_for_no_initializing_shards", String.valueOf(waitForNoInitializingShards));
    }
    params.put("wait_for_active_shards", String.valueOf(waitForActiveShards));
    params.put("wait_for_nodes", waitForNodes);
    params.put("wait_for_events", waitForEvents.name());
    FakeRestRequest restRequest = buildRestRequest(params);
    ClusterHealthRequest clusterHealthRequest = RestClusterHealthAction.fromRequest(restRequest);
    assertThat(clusterHealthRequest.indices().length, equalTo(1));
    assertThat(clusterHealthRequest.indices()[0], equalTo(index));
    assertThat(clusterHealthRequest.local(), equalTo(local));
    assertThat(clusterHealthRequest.masterNodeTimeout(), equalTo(TimeValue.parseTimeValue(masterTimeout, "test")));
    assertThat(clusterHealthRequest.timeout(), equalTo(TimeValue.parseTimeValue(timeout, "test")));
    assertThat(clusterHealthRequest.waitForStatus(), equalTo(waitForStatus));
    assertThat(clusterHealthRequest.waitForNoRelocatingShards(), equalTo(waitForNoRelocatingShards));
    assertThat(clusterHealthRequest.waitForNoInitializingShards(), equalTo(waitForNoInitializingShards));
    assertThat(clusterHealthRequest.waitForActiveShards(), equalTo(ActiveShardCount.parseString(String.valueOf(waitForActiveShards))));
    assertThat(clusterHealthRequest.waitForNodes(), equalTo(waitForNodes));
    assertThat(clusterHealthRequest.waitForEvents(), equalTo(waitForEvents));
}
Also used : ClusterHealthStatus(org.opensearch.cluster.health.ClusterHealthStatus) HashMap(java.util.HashMap) ClusterHealthRequest(org.opensearch.action.admin.cluster.health.ClusterHealthRequest) Priority(org.opensearch.common.Priority) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest)

Example 5 with Priority

use of org.opensearch.common.Priority in project OpenSearch by opensearch-project.

the class BlockMasterServiceOnMaster method startDisrupting.

@Override
public void startDisrupting() {
    disruptedNode = cluster.getMasterName();
    final String disruptionNodeCopy = disruptedNode;
    if (disruptionNodeCopy == null) {
        return;
    }
    ClusterService clusterService = cluster.getInstance(ClusterService.class, disruptionNodeCopy);
    if (clusterService == null) {
        return;
    }
    logger.info("blocking master service on node [{}]", disruptionNodeCopy);
    boolean success = disruptionLatch.compareAndSet(null, new CountDownLatch(1));
    assert success : "startDisrupting called without waiting on stopDisrupting to complete";
    final CountDownLatch started = new CountDownLatch(1);
    clusterService.getMasterService().submitStateUpdateTask("service_disruption_block", new ClusterStateUpdateTask() {

        @Override
        public Priority priority() {
            return Priority.IMMEDIATE;
        }

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            started.countDown();
            CountDownLatch latch = disruptionLatch.get();
            if (latch != null) {
                try {
                    latch.await();
                } catch (InterruptedException e) {
                    Throwables.rethrow(e);
                }
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            logger.error("unexpected error during disruption", e);
        }
    });
    try {
        started.await();
    } catch (InterruptedException e) {
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) ClusterService(org.opensearch.cluster.service.ClusterService) Priority(org.opensearch.common.Priority) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) CountDownLatch(java.util.concurrent.CountDownLatch)

Aggregations

Priority (org.opensearch.common.Priority)10 ClusterState (org.opensearch.cluster.ClusterState)5 ArrayList (java.util.ArrayList)4 Arrays (java.util.Arrays)4 Collections (java.util.Collections)4 HashMap (java.util.HashMap)4 HashSet (java.util.HashSet)4 Set (java.util.Set)4 CountDownLatch (java.util.concurrent.CountDownLatch)4 ActionListener (org.opensearch.action.ActionListener)4 List (java.util.List)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 AtomicLong (java.util.concurrent.atomic.AtomicLong)3 LogManager (org.apache.logging.log4j.LogManager)3 Version (org.opensearch.Version)3 ClusterStateUpdateTask (org.opensearch.cluster.ClusterStateUpdateTask)3 DiscoveryNodes (org.opensearch.cluster.node.DiscoveryNodes)3 ClusterService (org.opensearch.cluster.service.ClusterService)3 EnumSet (java.util.EnumSet)2 Map (java.util.Map)2