use of org.opensearch.cluster.block.ClusterBlock in project OpenSearch by opensearch-project.
the class MetadataIndexStateService method closeIndices.
/**
* Closes one or more indices.
*
* Closing indices is a 3 steps process: it first adds a write block to every indices to close, then waits for the operations on shards
* to be terminated and finally closes the indices by moving their state to CLOSE.
*/
public void closeIndices(final CloseIndexClusterStateUpdateRequest request, final ActionListener<CloseIndexResponse> listener) {
final Index[] concreteIndices = request.indices();
if (concreteIndices == null || concreteIndices.length == 0) {
throw new IllegalArgumentException("Index name is required");
}
List<String> writeIndices = new ArrayList<>();
SortedMap<String, IndexAbstraction> lookup = clusterService.state().metadata().getIndicesLookup();
for (Index index : concreteIndices) {
IndexAbstraction ia = lookup.get(index.getName());
if (ia != null && ia.getParentDataStream() != null && ia.getParentDataStream().getWriteIndex().getIndex().equals(index)) {
writeIndices.add(index.getName());
}
}
if (writeIndices.size() > 0) {
throw new IllegalArgumentException("cannot close the following data stream write indices [" + Strings.collectionToCommaDelimitedString(writeIndices) + "]");
}
clusterService.submitStateUpdateTask("add-block-index-to-close " + Arrays.toString(concreteIndices), new ClusterStateUpdateTask(Priority.URGENT) {
private final Map<Index, ClusterBlock> blockedIndices = new HashMap<>();
@Override
public ClusterState execute(final ClusterState currentState) {
return addIndexClosedBlocks(concreteIndices, blockedIndices, currentState);
}
@Override
public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
if (oldState == newState) {
assert blockedIndices.isEmpty() : "List of blocked indices is not empty but cluster state wasn't changed";
listener.onResponse(new CloseIndexResponse(true, false, Collections.emptyList()));
} else {
assert blockedIndices.isEmpty() == false : "List of blocked indices is empty but cluster state was changed";
threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new WaitForClosedBlocksApplied(blockedIndices, request, ActionListener.wrap(verifyResults -> clusterService.submitStateUpdateTask("close-indices", new ClusterStateUpdateTask(Priority.URGENT) {
private final List<IndexResult> indices = new ArrayList<>();
@Override
public ClusterState execute(final ClusterState currentState) throws Exception {
Tuple<ClusterState, Collection<IndexResult>> closingResult = closeRoutingTable(currentState, blockedIndices, verifyResults);
assert verifyResults.size() == closingResult.v2().size();
indices.addAll(closingResult.v2());
return allocationService.reroute(closingResult.v1(), "indices closed");
}
@Override
public void onFailure(final String source, final Exception e) {
listener.onFailure(e);
}
@Override
public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
final boolean acknowledged = indices.stream().noneMatch(IndexResult::hasFailures);
final String[] waitForIndices = indices.stream().filter(result -> result.hasFailures() == false).filter(result -> newState.routingTable().hasIndex(result.getIndex())).map(result -> result.getIndex().getName()).toArray(String[]::new);
if (waitForIndices.length > 0) {
activeShardsObserver.waitForActiveShards(waitForIndices, request.waitForActiveShards(), request.ackTimeout(), shardsAcknowledged -> {
if (shardsAcknowledged == false) {
logger.debug("[{}] indices closed, but the operation timed out while waiting " + "for enough shards to be started.", Arrays.toString(waitForIndices));
}
// acknowledged maybe be false but some indices may have been correctly
// closed, so
// we maintain a kind of coherency by overriding the shardsAcknowledged
// value
// (see ShardsAcknowledgedResponse constructor)
boolean shardsAcked = acknowledged ? shardsAcknowledged : false;
listener.onResponse(new CloseIndexResponse(acknowledged, shardsAcked, indices));
}, listener::onFailure);
} else {
listener.onResponse(new CloseIndexResponse(acknowledged, false, indices));
}
}
}), listener::onFailure)));
}
}
@Override
public void onFailure(final String source, final Exception e) {
listener.onFailure(e);
}
@Override
public TimeValue timeout() {
return request.masterNodeTimeout();
}
});
}
use of org.opensearch.cluster.block.ClusterBlock in project OpenSearch by opensearch-project.
the class MetadataIndexStateService method closeRoutingTable.
/**
* Step 3 - Move index states from OPEN to CLOSE in cluster state for indices that are ready for closing.
*/
static Tuple<ClusterState, Collection<IndexResult>> closeRoutingTable(final ClusterState currentState, final Map<Index, ClusterBlock> blockedIndices, final Map<Index, IndexResult> verifyResult) {
// Remove the index routing table of closed indices if the cluster is in a mixed version
// that does not support the replication of closed indices
final boolean removeRoutingTable = currentState.nodes().getMinNodeVersion().before(LegacyESVersion.V_7_2_0);
final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
final Set<String> closedIndices = new HashSet<>();
Map<Index, IndexResult> closingResults = new HashMap<>(verifyResult);
for (Map.Entry<Index, IndexResult> result : verifyResult.entrySet()) {
final Index index = result.getKey();
final boolean acknowledged = result.getValue().hasFailures() == false;
try {
if (acknowledged == false) {
logger.debug("verification of shards before closing {} failed [{}]", index, result);
continue;
}
final IndexMetadata indexMetadata = metadata.getSafe(index);
if (indexMetadata.getState() == IndexMetadata.State.CLOSE) {
logger.debug("verification of shards before closing {} succeeded but index is already closed", index);
assert currentState.blocks().hasIndexBlock(index.getName(), INDEX_CLOSED_BLOCK);
continue;
}
final ClusterBlock closingBlock = blockedIndices.get(index);
assert closingBlock != null;
if (currentState.blocks().hasIndexBlock(index.getName(), closingBlock) == false) {
// we should report error in this case as the index can be left as open.
closingResults.put(result.getKey(), new IndexResult(result.getKey(), new IllegalStateException("verification of shards before closing " + index + " succeeded but block has been removed in the meantime")));
logger.debug("verification of shards before closing {} succeeded but block has been removed in the meantime", index);
continue;
}
// Check if index closing conflicts with any running restores
Set<Index> restoringIndices = RestoreService.restoringIndices(currentState, singleton(index));
if (restoringIndices.isEmpty() == false) {
closingResults.put(result.getKey(), new IndexResult(result.getKey(), new IllegalStateException("verification of shards before closing " + index + " succeeded but index is being restored in the meantime")));
logger.debug("verification of shards before closing {} succeeded but index is being restored in the meantime", index);
continue;
}
// Check if index closing conflicts with any running snapshots
Set<Index> snapshottingIndices = SnapshotsService.snapshottingIndices(currentState, singleton(index));
if (snapshottingIndices.isEmpty() == false) {
closingResults.put(result.getKey(), new IndexResult(result.getKey(), new IllegalStateException("verification of shards before closing " + index + " succeeded but index is being snapshot in the meantime")));
logger.debug("verification of shards before closing {} succeeded but index is being snapshot in the meantime", index);
continue;
}
blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID);
blocks.addIndexBlock(index.getName(), INDEX_CLOSED_BLOCK);
final IndexMetadata.Builder updatedMetadata = IndexMetadata.builder(indexMetadata).state(IndexMetadata.State.CLOSE);
if (removeRoutingTable) {
metadata.put(updatedMetadata);
routingTable.remove(index.getName());
} else {
metadata.put(updatedMetadata.settingsVersion(indexMetadata.getSettingsVersion() + 1).settings(Settings.builder().put(indexMetadata.getSettings()).put(VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true)));
routingTable.addAsFromOpenToClose(metadata.getSafe(index));
}
logger.debug("closing index {} succeeded", index);
closedIndices.add(index.getName());
} catch (final IndexNotFoundException e) {
logger.debug("index {} has been deleted since it was blocked before closing, ignoring", index);
}
}
logger.info("completed closing of indices {}", closedIndices);
return Tuple.tuple(ClusterState.builder(currentState).blocks(blocks).metadata(metadata).routingTable(routingTable.build()).build(), closingResults.values());
}
use of org.opensearch.cluster.block.ClusterBlock in project OpenSearch by opensearch-project.
the class MetadataIndexStateService method finalizeBlock.
/**
* Finalizes the addition of blocks by turning the temporary UUID-based blocks into full blocks.
* @param currentState the cluster state to update
* @param blockedIndices the indices and their temporary UUID-based blocks to convert
* @param verifyResult the index-level results for adding the block
* @param block the full block to convert to
* @return the updated cluster state, as well as the (failed and successful) index-level results for adding the block
*/
static Tuple<ClusterState, Collection<AddBlockResult>> finalizeBlock(final ClusterState currentState, final Map<Index, ClusterBlock> blockedIndices, final Map<Index, AddBlockResult> verifyResult, final APIBlock block) {
final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
final Set<String> effectivelyBlockedIndices = new HashSet<>();
Map<Index, AddBlockResult> blockingResults = new HashMap<>(verifyResult);
for (Map.Entry<Index, AddBlockResult> result : verifyResult.entrySet()) {
final Index index = result.getKey();
final boolean acknowledged = result.getValue().hasFailures() == false;
try {
if (acknowledged == false) {
logger.debug("verification of shards before blocking {} failed [{}]", index, result);
continue;
}
final IndexMetadata indexMetadata = metadata.getSafe(index);
final ClusterBlock tempBlock = blockedIndices.get(index);
assert tempBlock != null;
assert tempBlock.uuid() != null;
final ClusterBlock currentBlock = currentState.blocks().getIndexBlockWithId(index.getName(), tempBlock.id());
if (currentBlock != null && currentBlock.equals(block.block)) {
logger.debug("verification of shards for {} succeeded, but block finalization already occurred" + " (possibly for another block) [{}]", index, result);
continue;
}
if (currentBlock == null || currentBlock.equals(tempBlock) == false) {
// we should report error in this case as the index can be left as open.
blockingResults.put(result.getKey(), new AddBlockResult(result.getKey(), new IllegalStateException("verification of shards before blocking " + index + " succeeded but block has been removed in the meantime")));
logger.debug("verification of shards before blocking {} succeeded but block has been removed in the meantime", index);
continue;
}
assert currentBlock != null && currentBlock.equals(tempBlock) && currentBlock.id() == block.block.id();
blocks.removeIndexBlockWithId(index.getName(), tempBlock.id());
blocks.addIndexBlock(index.getName(), block.block);
logger.debug("add block {} to index {} succeeded", block.block, index);
effectivelyBlockedIndices.add(index.getName());
} catch (final IndexNotFoundException e) {
logger.debug("index {} has been deleted since blocking it started, ignoring", index);
}
}
logger.info("completed adding block {} to indices {}", block.name, effectivelyBlockedIndices);
return Tuple.tuple(ClusterState.builder(currentState).blocks(blocks).metadata(metadata).routingTable(routingTable.build()).build(), blockingResults.values());
}
use of org.opensearch.cluster.block.ClusterBlock in project OpenSearch by opensearch-project.
the class TransportVerifyShardBeforeCloseActionTests method testUnavailableShardsMarkedAsStale.
public void testUnavailableShardsMarkedAsStale() throws Exception {
final String index = "test";
final ShardId shardId = new ShardId(index, "_na_", 0);
final int nbReplicas = randomIntBetween(1, 10);
final ShardRoutingState[] replicaStates = new ShardRoutingState[nbReplicas];
for (int i = 0; i < replicaStates.length; i++) {
replicaStates[i] = ShardRoutingState.STARTED;
}
final ClusterState clusterState = state(index, true, ShardRoutingState.STARTED, replicaStates);
setState(clusterService, clusterState);
IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index(index).shard(shardId.id());
final IndexMetadata indexMetadata = clusterState.getMetadata().index(index);
final ShardRouting primaryRouting = shardRoutingTable.primaryShard();
final long primaryTerm = indexMetadata.primaryTerm(0);
final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0);
final Set<String> trackedShards = shardRoutingTable.getAllAllocationIds();
List<ShardRouting> unavailableShards = randomSubsetOf(randomIntBetween(1, nbReplicas), shardRoutingTable.replicaShards());
IndexShardRoutingTable.Builder shardRoutingTableBuilder = new IndexShardRoutingTable.Builder(shardRoutingTable);
unavailableShards.forEach(shardRoutingTableBuilder::removeShard);
shardRoutingTable = shardRoutingTableBuilder.build();
final ReplicationGroup replicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
assertThat(replicationGroup.getUnavailableInSyncShards().size(), greaterThan(0));
final PlainActionFuture<PrimaryResult> listener = new PlainActionFuture<>();
TaskId taskId = new TaskId(clusterService.localNode().getId(), 0L);
TransportVerifyShardBeforeCloseAction.ShardRequest request = new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, clusterBlock, false, taskId);
ReplicationOperation.Replicas<TransportVerifyShardBeforeCloseAction.ShardRequest> proxy = action.newReplicasProxy();
ReplicationOperation<TransportVerifyShardBeforeCloseAction.ShardRequest, TransportVerifyShardBeforeCloseAction.ShardRequest, PrimaryResult> operation = new ReplicationOperation<>(request, createPrimary(primaryRouting, replicationGroup), listener, proxy, logger, threadPool, "test", primaryTerm, TimeValue.timeValueMillis(20), TimeValue.timeValueSeconds(60));
operation.execute();
final CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear();
assertThat(capturedRequests.length, equalTo(nbReplicas));
for (CapturingTransport.CapturedRequest capturedRequest : capturedRequests) {
final String actionName = capturedRequest.action;
if (actionName.startsWith(ShardStateAction.SHARD_FAILED_ACTION_NAME)) {
assertThat(capturedRequest.request, instanceOf(ShardStateAction.FailedShardEntry.class));
String allocationId = ((ShardStateAction.FailedShardEntry) capturedRequest.request).getAllocationId();
assertTrue(unavailableShards.stream().anyMatch(shardRouting -> shardRouting.allocationId().getId().equals(allocationId)));
transport.handleResponse(capturedRequest.requestId, TransportResponse.Empty.INSTANCE);
} else if (actionName.startsWith(TransportVerifyShardBeforeCloseAction.NAME)) {
assertThat(capturedRequest.request, instanceOf(ConcreteShardRequest.class));
String allocationId = ((ConcreteShardRequest) capturedRequest.request).getTargetAllocationID();
assertFalse(unavailableShards.stream().anyMatch(shardRouting -> shardRouting.allocationId().getId().equals(allocationId)));
assertTrue(inSyncAllocationIds.stream().anyMatch(inSyncAllocationId -> inSyncAllocationId.equals(allocationId)));
transport.handleResponse(capturedRequest.requestId, new TransportReplicationAction.ReplicaResponse(0L, 0L));
} else {
fail("Test does not support action " + capturedRequest.action);
}
}
final ReplicationResponse.ShardInfo shardInfo = listener.get().getShardInfo();
assertThat(shardInfo.getFailed(), equalTo(0));
assertThat(shardInfo.getFailures(), arrayWithSize(0));
assertThat(shardInfo.getSuccessful(), equalTo(1 + nbReplicas - unavailableShards.size()));
}
use of org.opensearch.cluster.block.ClusterBlock in project OpenSearch by opensearch-project.
the class TransportVerifyShardBeforeCloseActionTests method executeOnPrimaryOrReplica.
private void executeOnPrimaryOrReplica(boolean phase1) throws Throwable {
final TaskId taskId = new TaskId("_node_id", randomNonNegativeLong());
final TransportVerifyShardBeforeCloseAction.ShardRequest request = new TransportVerifyShardBeforeCloseAction.ShardRequest(indexShard.shardId(), clusterBlock, phase1, taskId);
final PlainActionFuture<Void> res = PlainActionFuture.newFuture();
action.shardOperationOnPrimary(request, indexShard, ActionListener.wrap(r -> {
assertNotNull(r);
res.onResponse(null);
}, res::onFailure));
try {
res.get();
} catch (InterruptedException e) {
throw new AssertionError(e);
} catch (ExecutionException e) {
throw e.getCause();
}
}
Aggregations