Search in sources :

Example 1 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class ShrinkIndexIT method testCreateShrinkIndex.

public void testCreateShrinkIndex() {
    internalCluster().ensureAtLeastNumDataNodes(2);
    Version version = VersionUtils.randomVersion(random());
    prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7)).put("index.version.created", version)).get();
    final int docs = randomIntBetween(0, 128);
    for (int i = 0; i < docs; i++) {
        client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
    }
    ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
    assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
    DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
    // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
    // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
    // to the require._name below.
    ensureGreen();
    // relocate all shards to one node such that we can merge it.
    client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.routing.allocation.require._name", discoveryNodes[0].getName()).put("index.blocks.write", true)).get();
    ensureGreen();
    final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get();
    // disable rebalancing to be able to capture the right stats. balancing can move the target primary
    // making it hard to pin point the source shards.
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")).get();
    // now merge source into a single shard index
    final boolean createWithReplicas = randomBoolean();
    assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name").build()).get());
    ensureGreen();
    // resolve true merge node - this is not always the node we required as all shards may be on another node
    final ClusterState state = client().admin().cluster().prepareState().get().getState();
    DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId());
    logger.info("merge node {}", mergeNode);
    final long maxSeqNo = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getSeqNoStats).mapToLong(SeqNoStats::getMaxSeqNo).max().getAsLong();
    final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards()).filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())).map(ShardStats::getStats).map(CommonStats::getSegments).mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp).max().getAsLong();
    final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get();
    for (final ShardStats shardStats : targetStats.getShards()) {
        final SeqNoStats seqNoStats = shardStats.getSeqNoStats();
        final ShardRouting shardRouting = shardStats.getShardRouting();
        assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo));
        assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo));
        assertThat("failed on " + shardRouting, shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(), equalTo(maxUnsafeAutoIdTimestamp));
    }
    final int size = docs > 0 ? 2 * docs : 1;
    assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
    if (createWithReplicas == false) {
        // bump replicas
        client().admin().indices().prepareUpdateSettings("target").setSettings(Settings.builder().put("index.number_of_replicas", 1)).get();
        ensureGreen();
        assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
    }
    for (int i = docs; i < 2 * docs; i++) {
        client().prepareIndex("target").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
    }
    flushAndRefresh();
    assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 2 * docs);
    assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs);
    GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
    assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
    // clean up
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)).get();
}
Also used : CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) Arrays(java.util.Arrays) EnableAllocationDecider(org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider) ClusterStateResponse(org.opensearch.action.admin.cluster.state.ClusterStateResponse) Version(org.opensearch.Version) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) OpenSearchAssertions.assertHitCount(org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount) ClusterRerouteResponse(org.opensearch.action.admin.cluster.reroute.ClusterRerouteResponse) Map(java.util.Map) SortField(org.apache.lucene.search.SortField) SegmentsStats(org.opensearch.index.engine.SegmentsStats) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) Sort(org.apache.lucene.search.Sort) Index(org.opensearch.index.Index) IndicesService(org.opensearch.indices.IndicesService) SortedSetSortField(org.apache.lucene.search.SortedSetSortField) Settings(org.opensearch.common.settings.Settings) InternalClusterInfoService(org.opensearch.cluster.InternalClusterInfoService) Matchers.equalTo(org.hamcrest.Matchers.equalTo) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) IndicesStatsResponse(org.opensearch.action.admin.indices.stats.IndicesStatsResponse) XContentType(org.opensearch.common.xcontent.XContentType) OpenSearchIntegTestCase(org.opensearch.test.OpenSearchIntegTestCase) Matchers.containsString(org.hamcrest.Matchers.containsString) IntStream(java.util.stream.IntStream) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ClusterInfoService(org.opensearch.cluster.ClusterInfoService) Priority(org.opensearch.common.Priority) InternalTestCluster(org.opensearch.test.InternalTestCluster) ClusterState(org.opensearch.cluster.ClusterState) IndexShard(org.opensearch.index.shard.IndexShard) VersionUtils(org.opensearch.test.VersionUtils) Murmur3HashFunction(org.opensearch.cluster.routing.Murmur3HashFunction) OpenSearchAssertions.assertAcked(org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked) ShardSegments(org.opensearch.action.admin.indices.segments.ShardSegments) ResizeType(org.opensearch.action.admin.indices.shrink.ResizeType) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) IndexShardSegments(org.opensearch.action.admin.indices.segments.IndexShardSegments) GetSettingsResponse(org.opensearch.action.admin.indices.settings.get.GetSettingsResponse) IndexService(org.opensearch.index.IndexService) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ClusterStateRequest(org.opensearch.action.admin.cluster.state.ClusterStateRequest) Constants(org.apache.lucene.util.Constants) SortedSetSelector(org.apache.lucene.search.SortedSetSelector) RoutingTable(org.opensearch.cluster.routing.RoutingTable) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) IndexRequest(org.opensearch.action.index.IndexRequest) IndicesSegmentResponse(org.opensearch.action.admin.indices.segments.IndicesSegmentResponse) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) IndicesStatsResponse(org.opensearch.action.admin.indices.stats.IndicesStatsResponse) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) GetSettingsResponse(org.opensearch.action.admin.indices.settings.get.GetSettingsResponse) Matchers.containsString(org.hamcrest.Matchers.containsString) SegmentsStats(org.opensearch.index.engine.SegmentsStats) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) Version(org.opensearch.Version) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 2 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class RestoreClusterStateListener method clusterChanged.

@Override
public void clusterChanged(ClusterChangedEvent changedEvent) {
    final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid);
    final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid);
    if (prevEntry == null) {
        // When there is a master failure after a restore has been started, this listener might not be registered
        // on the current master and as such it might miss some intermediary cluster states due to batching.
        // Clean up listener in that case and acknowledge completion of restore operation to client.
        clusterService.removeListener(this);
        listener.onResponse(new RestoreSnapshotResponse((RestoreInfo) null));
    } else if (newEntry == null) {
        clusterService.removeListener(this);
        ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards = prevEntry.shards();
        assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state();
        assert RestoreService.completed(shards) : "expected all restore entries to be completed";
        RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), prevEntry.indices(), shards.size(), shards.size() - RestoreService.failedShards(shards));
        RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri);
        logger.debug("restore of [{}] completed", prevEntry.snapshot().getSnapshotId());
        listener.onResponse(response);
    } else {
    // restore not completed yet, wait for next cluster state update
    }
}
Also used : RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RestoreInfo(org.opensearch.snapshots.RestoreInfo) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap)

Example 3 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class TransportClusterRerouteAction method verifyThenSubmitUpdate.

private void verifyThenSubmitUpdate(ClusterRerouteRequest request, ActionListener<ClusterRerouteResponse> listener, Map<String, List<AbstractAllocateAllocationCommand>> stalePrimaryAllocations) {
    transportService.sendRequest(transportService.getLocalNode(), IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(stalePrimaryAllocations.keySet().toArray(Strings.EMPTY_ARRAY)), new ActionListenerResponseHandler<>(ActionListener.wrap(response -> {
        ImmutableOpenMap<String, ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>>> status = response.getStoreStatuses();
        Exception e = null;
        for (Map.Entry<String, List<AbstractAllocateAllocationCommand>> entry : stalePrimaryAllocations.entrySet()) {
            final String index = entry.getKey();
            final ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> indexStatus = status.get(index);
            if (indexStatus == null) {
                // request. We ignore it here since the relevant exception will be thrown by the reroute action later on.
                continue;
            }
            for (AbstractAllocateAllocationCommand command : entry.getValue()) {
                final List<IndicesShardStoresResponse.StoreStatus> shardStatus = indexStatus.get(command.shardId());
                if (shardStatus == null || shardStatus.isEmpty()) {
                    e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on any node"));
                } else if (shardStatus.stream().noneMatch(storeStatus -> {
                    final DiscoveryNode node = storeStatus.getNode();
                    final String nodeInCommand = command.node();
                    return nodeInCommand.equals(node.getName()) || nodeInCommand.equals(node.getId());
                })) {
                    e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on node [" + command.node() + ']'));
                }
            }
        }
        if (e == null) {
            submitStateUpdate(request, listener);
        } else {
            listener.onFailure(e);
        }
    }, listener::onFailure), IndicesShardStoresResponse::new));
}
Also used : ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) ThreadPool(org.opensearch.threadpool.ThreadPool) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) Priority(org.opensearch.common.Priority) HashMap(java.util.HashMap) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) AllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocationCommand) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) IndicesShardStoresAction(org.opensearch.action.admin.indices.shards.IndicesShardStoresAction) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) Strings(org.opensearch.common.Strings) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Map(java.util.Map) AckedClusterStateUpdateTask(org.opensearch.cluster.AckedClusterStateUpdateTask) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) StreamInput(org.opensearch.common.io.stream.StreamInput) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) RoutingExplanations(org.opensearch.cluster.routing.allocation.RoutingExplanations) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) ExceptionsHelper(org.opensearch.ExceptionsHelper) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) TransportService(org.opensearch.transport.TransportService) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) ClusterService(org.opensearch.cluster.service.ClusterService) LogManager(org.apache.logging.log4j.LogManager) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) IndicesShardStoresRequest(org.opensearch.action.admin.indices.shards.IndicesShardStoresRequest) IndicesShardStoresResponse(org.opensearch.action.admin.indices.shards.IndicesShardStoresResponse) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AbstractAllocateAllocationCommand(org.opensearch.cluster.routing.allocation.command.AbstractAllocateAllocationCommand) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IOException(java.io.IOException) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableOpenIntMap(org.opensearch.common.collect.ImmutableOpenIntMap)

Example 4 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class DiskThresholdMonitor method onNewInfo.

public void onNewInfo(ClusterInfo info) {
    // all ClusterInfo updates are processed and never ignored
    if (checkInProgress.compareAndSet(false, true) == false) {
        logger.info("skipping monitor as a check is already in progress");
        return;
    }
    final ImmutableOpenMap<String, DiskUsage> usages = info.getNodeLeastAvailableDiskUsages();
    if (usages == null) {
        logger.trace("skipping monitor as no disk usage information is available");
        checkFinished();
        return;
    }
    logger.trace("processing new cluster info");
    boolean reroute = false;
    String explanation = "";
    final long currentTimeMillis = currentTimeMillisSupplier.getAsLong();
    // Clean up nodes that have been removed from the cluster
    final ObjectLookupContainer<String> nodes = usages.keys();
    cleanUpRemovedNodes(nodes, nodesOverLowThreshold);
    cleanUpRemovedNodes(nodes, nodesOverHighThreshold);
    cleanUpRemovedNodes(nodes, nodesOverHighThresholdAndRelocating);
    final ClusterState state = clusterStateSupplier.get();
    final Set<String> indicesToMarkReadOnly = new HashSet<>();
    RoutingNodes routingNodes = state.getRoutingNodes();
    Set<String> indicesNotToAutoRelease = new HashSet<>();
    markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesNotToAutoRelease);
    final List<DiskUsage> usagesOverHighThreshold = new ArrayList<>();
    for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
        final String node = entry.key;
        final DiskUsage usage = entry.value;
        final RoutingNode routingNode = routingNodes.node(node);
        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
            nodesOverLowThreshold.add(node);
            nodesOverHighThreshold.add(node);
            nodesOverHighThresholdAndRelocating.remove(node);
            if (routingNode != null) {
                // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                for (ShardRouting routing : routingNode) {
                    String indexName = routing.index().getName();
                    indicesToMarkReadOnly.add(indexName);
                    indicesNotToAutoRelease.add(indexName);
                }
            }
            logger.warn("flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only", diskThresholdSettings.describeFloodStageThreshold(), usage);
            continue;
        }
        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
            if (routingNode != null) {
                // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                for (ShardRouting routing : routingNode) {
                    String indexName = routing.index().getName();
                    indicesNotToAutoRelease.add(indexName);
                }
            }
        }
        final long reservedSpace = info.getReservedSpace(usage.getNodeId(), usage.getPath()).getTotal();
        final DiskUsage usageWithReservedSpace = new DiskUsage(usage.getNodeId(), usage.getNodeName(), usage.getPath(), usage.getTotalBytes(), Math.max(0L, usage.getFreeBytes() - reservedSpace));
        if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
            nodesOverLowThreshold.add(node);
            nodesOverHighThreshold.add(node);
            if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                reroute = true;
                explanation = "high disk watermark exceeded on one or more nodes";
                usagesOverHighThreshold.add(usage);
            // will log about this node when the reroute completes
            } else {
                logger.debug("high disk watermark exceeded on {} but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
            }
        } else if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdLow()) {
            nodesOverHighThresholdAndRelocating.remove(node);
            final boolean wasUnderLowThreshold = nodesOverLowThreshold.add(node);
            final boolean wasOverHighThreshold = nodesOverHighThreshold.remove(node);
            assert (wasUnderLowThreshold && wasOverHighThreshold) == false;
            if (wasUnderLowThreshold) {
                logger.info("low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", diskThresholdSettings.describeLowThreshold(), usage);
            } else if (wasOverHighThreshold) {
                logger.info("high disk watermark [{}] no longer exceeded on {}, but low disk watermark [{}] is still exceeded", diskThresholdSettings.describeHighThreshold(), usage, diskThresholdSettings.describeLowThreshold());
            }
        } else {
            nodesOverHighThresholdAndRelocating.remove(node);
            if (nodesOverLowThreshold.contains(node)) {
                // if we reroute now.
                if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                    reroute = true;
                    explanation = "one or more nodes has gone under the high or low watermark";
                    nodesOverLowThreshold.remove(node);
                    nodesOverHighThreshold.remove(node);
                    logger.info("low disk watermark [{}] no longer exceeded on {}", diskThresholdSettings.describeLowThreshold(), usage);
                } else {
                    logger.debug("{} has gone below a disk threshold, but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
                }
            }
        }
    }
    final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 3);
    if (reroute) {
        logger.debug("rerouting shards: [{}]", explanation);
        rerouteService.reroute("disk threshold monitor", Priority.HIGH, ActionListener.wrap(reroutedClusterState -> {
            for (DiskUsage diskUsage : usagesOverHighThreshold) {
                final RoutingNode routingNode = reroutedClusterState.getRoutingNodes().node(diskUsage.getNodeId());
                final DiskUsage usageIncludingRelocations;
                final long relocatingShardsSize;
                if (routingNode != null) {
                    // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                    relocatingShardsSize = sizeOfRelocatingShards(routingNode, diskUsage, info, reroutedClusterState);
                    usageIncludingRelocations = new DiskUsage(diskUsage.getNodeId(), diskUsage.getNodeName(), diskUsage.getPath(), diskUsage.getTotalBytes(), diskUsage.getFreeBytes() - relocatingShardsSize);
                } else {
                    usageIncludingRelocations = diskUsage;
                    relocatingShardsSize = 0L;
                }
                if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usageIncludingRelocations.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
                    nodesOverHighThresholdAndRelocating.remove(diskUsage.getNodeId());
                    logger.warn("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to continue to exceed " + "the high disk watermark when these relocations are complete", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                } else if (nodesOverHighThresholdAndRelocating.add(diskUsage.getNodeId())) {
                    logger.info("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to be below the high " + "disk watermark when these relocations are complete", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                } else {
                    logger.debug("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                }
            }
            setLastRunTimeMillis();
            listener.onResponse(null);
        }, e -> {
            logger.debug("reroute failed", e);
            setLastRunTimeMillis();
            listener.onFailure(e);
        }));
    } else {
        logger.trace("no reroute required");
        listener.onResponse(null);
    }
    final Set<String> indicesToAutoRelease = StreamSupport.stream(state.routingTable().indicesRouting().spliterator(), false).map(c -> c.key).filter(index -> indicesNotToAutoRelease.contains(index) == false).filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)).collect(Collectors.toSet());
    if (indicesToAutoRelease.isEmpty() == false) {
        if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
            logger.info("releasing read-only-allow-delete block on indices: [{}]", indicesToAutoRelease);
            updateIndicesReadOnly(indicesToAutoRelease, listener, false);
        } else {
            deprecationLogger.deprecate(DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY.replace(".", "_"), "[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, LegacyESVersion.V_7_4_0.major + 1);
            logger.debug("[{}] disabled, not releasing read-only-allow-delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
            listener.onResponse(null);
        }
    } else {
        logger.trace("no auto-release required");
        listener.onResponse(null);
    }
    indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
    logger.trace("marking indices as read-only: [{}]", indicesToMarkReadOnly);
    if (indicesToMarkReadOnly.isEmpty() == false) {
        updateIndicesReadOnly(indicesToMarkReadOnly, listener, true);
    } else {
        listener.onResponse(null);
    }
}
Also used : ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) LongSupplier(java.util.function.LongSupplier) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Priority(org.opensearch.common.Priority) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Supplier(java.util.function.Supplier) Strings(org.opensearch.common.Strings) ArrayList(java.util.ArrayList) DeprecationLogger(org.opensearch.common.logging.DeprecationLogger) HashSet(java.util.HashSet) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) RerouteService(org.opensearch.cluster.routing.RerouteService) RoutingNodes(org.opensearch.cluster.routing.RoutingNodes) StreamSupport(java.util.stream.StreamSupport) ActionListener(org.opensearch.action.ActionListener) DiskUsage(org.opensearch.cluster.DiskUsage) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Client(org.opensearch.client.Client) DiskThresholdDecider(org.opensearch.cluster.routing.allocation.decider.DiskThresholdDecider) ClusterInfo(org.opensearch.cluster.ClusterInfo) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) Collectors(java.util.stream.Collectors) ShardRouting(org.opensearch.cluster.routing.ShardRouting) AtomicLong(java.util.concurrent.atomic.AtomicLong) Sets(org.opensearch.common.util.set.Sets) List(java.util.List) Logger(org.apache.logging.log4j.Logger) RoutingNode(org.opensearch.cluster.routing.RoutingNode) LogManager(org.apache.logging.log4j.LogManager) ObjectLookupContainer(com.carrotsearch.hppc.ObjectLookupContainer) ClusterState(org.opensearch.cluster.ClusterState) RoutingNodes(org.opensearch.cluster.routing.RoutingNodes) ArrayList(java.util.ArrayList) DiskUsage(org.opensearch.cluster.DiskUsage) RoutingNode(org.opensearch.cluster.routing.RoutingNode) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) ShardRouting(org.opensearch.cluster.routing.ShardRouting) HashSet(java.util.HashSet)

Example 5 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class SnapshotsService method shards.

/**
 * Calculates the assignment of shards to data nodes for a new snapshot based on the given cluster state and the
 * indices that should be included in the snapshot.
 *
 * @param indices             Indices to snapshot
 * @param useShardGenerations whether to write {@link ShardGenerations} during the snapshot
 * @return list of shard to be included into current snapshot
 */
private static ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards(SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, Metadata metadata, RoutingTable routingTable, List<IndexId> indices, boolean useShardGenerations, RepositoryData repositoryData, String repoName) {
    ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
    final ShardGenerations shardGenerations = repositoryData.shardGenerations();
    final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(repoName, snapshotsInProgress.entries());
    final boolean readyToExecute = deletionsInProgress == null || deletionsInProgress.getEntries().stream().noneMatch(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED);
    for (IndexId index : indices) {
        final String indexName = index.getName();
        final boolean isNewIndex = repositoryData.getIndices().containsKey(indexName) == false;
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            // The index was deleted before we managed to start the snapshot - mark it as missing.
            builder.put(new ShardId(indexName, IndexMetadata.INDEX_UUID_NA_VALUE, 0), ShardSnapshotStatus.MISSING);
        } else {
            final IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
            for (int i = 0; i < indexMetadata.getNumberOfShards(); i++) {
                final ShardId shardId = indexRoutingTable.shard(i).shardId();
                final String shardRepoGeneration;
                if (useShardGenerations) {
                    final String inFlightGeneration = inFlightShardStates.generationForShard(index, shardId.id(), shardGenerations);
                    if (inFlightGeneration == null && isNewIndex) {
                        assert shardGenerations.getShardGen(index, shardId.getId()) == null : "Found shard generation for new index [" + index + "]";
                        shardRepoGeneration = ShardGenerations.NEW_SHARD_GEN;
                    } else {
                        shardRepoGeneration = inFlightGeneration;
                    }
                } else {
                    shardRepoGeneration = null;
                }
                final ShardSnapshotStatus shardSnapshotStatus;
                if (indexRoutingTable == null) {
                    shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(null, ShardState.MISSING, "missing routing table", shardRepoGeneration);
                } else {
                    ShardRouting primary = indexRoutingTable.shard(i).primaryShard();
                    if (readyToExecute == false || inFlightShardStates.isActive(indexName, i)) {
                        shardSnapshotStatus = ShardSnapshotStatus.UNASSIGNED_QUEUED;
                    } else if (primary == null || !primary.assignedToNode()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(null, ShardState.MISSING, "primary shard is not allocated", shardRepoGeneration);
                    } else if (primary.relocating() || primary.initializing()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.WAITING, shardRepoGeneration);
                    } else if (!primary.started()) {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), ShardState.MISSING, "primary shard hasn't been started yet", shardRepoGeneration);
                    } else {
                        shardSnapshotStatus = new ShardSnapshotStatus(primary.currentNodeId(), shardRepoGeneration);
                    }
                }
                builder.put(shardId, shardSnapshotStatus);
            }
        }
    }
    return builder.build();
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) Collections.unmodifiableList(java.util.Collections.unmodifiableList) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) ClusterStateApplier(org.opensearch.cluster.ClusterStateApplier) Regex(org.opensearch.common.regex.Regex) Strings(org.opensearch.common.Strings) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) ObjectCursor(com.carrotsearch.hppc.cursors.ObjectCursor) TransportService(org.opensearch.transport.TransportService) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) ActionFilters(org.opensearch.action.support.ActionFilters) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) ShardState(org.opensearch.cluster.SnapshotsInProgress.ShardState) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) ClusterStateUpdateTask(org.opensearch.cluster.ClusterStateUpdateTask) StepListener(org.opensearch.action.StepListener) State(org.opensearch.cluster.SnapshotsInProgress.State) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) RepositoriesService(org.opensearch.repositories.RepositoriesService) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) TransportMasterNodeAction(org.opensearch.action.support.master.TransportMasterNodeAction) ArrayList(java.util.ArrayList) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) RepositoryCleanupInProgress(org.opensearch.cluster.RepositoryCleanupInProgress) RepositoriesMetadata(org.opensearch.cluster.metadata.RepositoriesMetadata) Executor(java.util.concurrent.Executor) IOException(java.io.IOException) DeleteSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest) ClusterService(org.opensearch.cluster.service.ClusterService) RestoreInProgress(org.opensearch.cluster.RestoreInProgress) RoutingTable(org.opensearch.cluster.routing.RoutingTable) SnapshotsInProgress.completed(org.opensearch.cluster.SnapshotsInProgress.completed) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ShardGenerations(org.opensearch.repositories.ShardGenerations) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexId(org.opensearch.repositories.IndexId) Locale(java.util.Locale) NotMasterException(org.opensearch.cluster.NotMasterException) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) RepositoryException(org.opensearch.repositories.RepositoryException) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) List(java.util.List) Optional(java.util.Optional) ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) CloneSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest) HashMap(java.util.HashMap) SnapshotDeletionsInProgress(org.opensearch.cluster.SnapshotDeletionsInProgress) Deque(java.util.Deque) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) UUIDs(org.opensearch.common.UUIDs) LinkedList(java.util.LinkedList) StreamInput(org.opensearch.common.io.stream.StreamInput) RepositoryData(org.opensearch.repositories.RepositoryData) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) IndexId(org.opensearch.repositories.IndexId) IndexRoutingTable(org.opensearch.cluster.routing.IndexRoutingTable) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) ShardGenerations(org.opensearch.repositories.ShardGenerations) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) RepositoryShardId(org.opensearch.repositories.RepositoryShardId) ShardId(org.opensearch.index.shard.ShardId) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) ShardSnapshotStatus(org.opensearch.cluster.SnapshotsInProgress.ShardSnapshotStatus) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Aggregations

ImmutableOpenMap (org.opensearch.common.collect.ImmutableOpenMap)43 ClusterState (org.opensearch.cluster.ClusterState)23 Settings (org.opensearch.common.settings.Settings)21 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)19 HashSet (java.util.HashSet)18 Matchers.containsString (org.hamcrest.Matchers.containsString)17 RoutingTable (org.opensearch.cluster.routing.RoutingTable)17 ShardId (org.opensearch.index.shard.ShardId)17 List (java.util.List)16 Metadata (org.opensearch.cluster.metadata.Metadata)15 ClusterInfo (org.opensearch.cluster.ClusterInfo)14 DiskUsage (org.opensearch.cluster.DiskUsage)14 IndexRoutingTable (org.opensearch.cluster.routing.IndexRoutingTable)14 IndexShardRoutingTable (org.opensearch.cluster.routing.IndexShardRoutingTable)14 ShardRouting (org.opensearch.cluster.routing.ShardRouting)14 ClusterSettings (org.opensearch.common.settings.ClusterSettings)14 ArrayList (java.util.ArrayList)13 Arrays (java.util.Arrays)13 HashMap (java.util.HashMap)13 Map (java.util.Map)13