use of org.opensearch.common.collect.ImmutableOpenIntMap in project OpenSearch by opensearch-project.
the class TransportClusterRerouteAction method verifyThenSubmitUpdate.
private void verifyThenSubmitUpdate(ClusterRerouteRequest request, ActionListener<ClusterRerouteResponse> listener, Map<String, List<AbstractAllocateAllocationCommand>> stalePrimaryAllocations) {
transportService.sendRequest(transportService.getLocalNode(), IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(stalePrimaryAllocations.keySet().toArray(Strings.EMPTY_ARRAY)), new ActionListenerResponseHandler<>(ActionListener.wrap(response -> {
ImmutableOpenMap<String, ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>>> status = response.getStoreStatuses();
Exception e = null;
for (Map.Entry<String, List<AbstractAllocateAllocationCommand>> entry : stalePrimaryAllocations.entrySet()) {
final String index = entry.getKey();
final ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> indexStatus = status.get(index);
if (indexStatus == null) {
// request. We ignore it here since the relevant exception will be thrown by the reroute action later on.
continue;
}
for (AbstractAllocateAllocationCommand command : entry.getValue()) {
final List<IndicesShardStoresResponse.StoreStatus> shardStatus = indexStatus.get(command.shardId());
if (shardStatus == null || shardStatus.isEmpty()) {
e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on any node"));
} else if (shardStatus.stream().noneMatch(storeStatus -> {
final DiscoveryNode node = storeStatus.getNode();
final String nodeInCommand = command.node();
return nodeInCommand.equals(node.getName()) || nodeInCommand.equals(node.getId());
})) {
e = ExceptionsHelper.useOrSuppress(e, new IllegalArgumentException("No data for shard [" + command.shardId() + "] of index [" + index + "] found on node [" + command.node() + ']'));
}
}
}
if (e == null) {
submitStateUpdate(request, listener);
} else {
listener.onFailure(e);
}
}, listener::onFailure), IndicesShardStoresResponse::new));
}
use of org.opensearch.common.collect.ImmutableOpenIntMap in project OpenSearch by opensearch-project.
the class PrimaryAllocationIT method testForceStaleReplicaToBePromotedToPrimary.
public void testForceStaleReplicaToBePromotedToPrimary() throws Exception {
logger.info("--> starting 3 nodes, 1 master, 2 data");
String master = internalCluster().startMasterOnlyNode(Settings.EMPTY);
internalCluster().startDataOnlyNodes(2);
assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get());
ensureGreen();
Set<String> historyUUIDs = Arrays.stream(client().admin().indices().prepareStats("test").clear().get().getShards()).map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet());
createStaleReplicaScenario(master);
if (randomBoolean()) {
assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(0));
}
// if true, use stale replica, otherwise a completely empty copy
boolean useStaleReplica = randomBoolean();
logger.info("--> explicitly promote old primary shard");
final String idxName = "test";
ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> storeStatuses = client().admin().indices().prepareShardStores(idxName).get().getStoreStatuses().get(idxName);
ClusterRerouteRequestBuilder rerouteBuilder = client().admin().cluster().prepareReroute();
for (IntObjectCursor<List<IndicesShardStoresResponse.StoreStatus>> shardStoreStatuses : storeStatuses) {
int shardId = shardStoreStatuses.key;
IndicesShardStoresResponse.StoreStatus storeStatus = randomFrom(shardStoreStatuses.value);
logger.info("--> adding allocation command for shard {}", shardId);
// force allocation based on node id
if (useStaleReplica) {
rerouteBuilder.add(new AllocateStalePrimaryAllocationCommand(idxName, shardId, storeStatus.getNode().getId(), true));
} else {
rerouteBuilder.add(new AllocateEmptyPrimaryAllocationCommand(idxName, shardId, storeStatus.getNode().getId(), true));
}
}
final Set<String> expectedAllocationIds = useStaleReplica ? Collections.singleton(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID) : Collections.emptySet();
final CountDownLatch clusterStateChangeLatch = new CountDownLatch(1);
final ClusterStateListener clusterStateListener = event -> {
final Set<String> allocationIds = event.state().metadata().index(idxName).inSyncAllocationIds(0);
if (expectedAllocationIds.equals(allocationIds)) {
clusterStateChangeLatch.countDown();
}
logger.info("expected allocation ids: {} actual allocation ids: {}", expectedAllocationIds, allocationIds);
};
final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, master);
clusterService.addListener(clusterStateListener);
rerouteBuilder.get();
assertTrue(clusterStateChangeLatch.await(30, TimeUnit.SECONDS));
clusterService.removeListener(clusterStateListener);
logger.info("--> check that the stale primary shard gets allocated and that documents are available");
ensureYellow(idxName);
if (useStaleReplica == false) {
// When invoking AllocateEmptyPrimaryAllocationCommand, due to the UnassignedInfo.Reason being changed to INDEX_CREATION,
// its possible that the shard has not completed initialization, even though the cluster health is yellow, so the
// search can throw an "all shards failed" exception. We will wait until the shard initialization has completed before
// verifying the search hit count.
assertBusy(() -> assertTrue(client().admin().cluster().prepareState().get().getState().routingTable().index(idxName).allPrimaryShardsActive()));
}
ShardStats[] shardStats = client().admin().indices().prepareStats("test").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED).get().getShards();
for (ShardStats shardStat : shardStats) {
assertThat(shardStat.getCommitStats().getNumDocs(), equalTo(useStaleReplica ? 1 : 0));
}
// allocation id of old primary was cleaned from the in-sync set
final ClusterState state = client().admin().cluster().prepareState().get().getState();
assertEquals(Collections.singleton(state.routingTable().index(idxName).shard(0).primary.allocationId().getId()), state.metadata().index(idxName).inSyncAllocationIds(0));
Set<String> newHistoryUUIds = Stream.of(shardStats).map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet());
assertThat(newHistoryUUIds, everyItem(is(not(in(historyUUIDs)))));
assertThat(newHistoryUUIds, hasSize(1));
}
use of org.opensearch.common.collect.ImmutableOpenIntMap in project OpenSearch by opensearch-project.
the class IndicesShardStoreResponseTests method testBasicSerialization.
public void testBasicSerialization() throws Exception {
ImmutableOpenMap.Builder<String, ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>>> indexStoreStatuses = ImmutableOpenMap.builder();
List<IndicesShardStoresResponse.Failure> failures = new ArrayList<>();
ImmutableOpenIntMap.Builder<List<IndicesShardStoresResponse.StoreStatus>> storeStatuses = ImmutableOpenIntMap.builder();
DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
List<IndicesShardStoresResponse.StoreStatus> storeStatusList = new ArrayList<>();
storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null));
storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node2, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null));
storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node1, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED, new IOException("corrupted")));
storeStatuses.put(0, storeStatusList);
storeStatuses.put(1, storeStatusList);
ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> storesMap = storeStatuses.build();
indexStoreStatuses.put("test", storesMap);
indexStoreStatuses.put("test2", storesMap);
failures.add(new IndicesShardStoresResponse.Failure("node1", "test", 3, new NodeDisconnectedException(node1, "")));
IndicesShardStoresResponse storesResponse = new IndicesShardStoresResponse(indexStoreStatuses.build(), Collections.unmodifiableList(failures));
XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
contentBuilder.startObject();
storesResponse.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
contentBuilder.endObject();
BytesReference bytes = BytesReference.bytes(contentBuilder);
try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
Map<String, Object> map = parser.map();
List<?> failureList = (List<?>) map.get("failures");
assertThat(failureList.size(), equalTo(1));
@SuppressWarnings("unchecked") Map<String, ?> failureMap = (Map<String, ?>) failureList.get(0);
assertThat(failureMap.containsKey("index"), equalTo(true));
assertThat(((String) failureMap.get("index")), equalTo("test"));
assertThat(failureMap.containsKey("shard"), equalTo(true));
assertThat(((int) failureMap.get("shard")), equalTo(3));
assertThat(failureMap.containsKey("node"), equalTo(true));
assertThat(((String) failureMap.get("node")), equalTo("node1"));
@SuppressWarnings("unchecked") Map<String, Object> indices = (Map<String, Object>) map.get("indices");
for (String index : new String[] { "test", "test2" }) {
assertThat(indices.containsKey(index), equalTo(true));
@SuppressWarnings("unchecked") Map<String, Object> shards = ((Map<String, Object>) ((Map<String, Object>) indices.get(index)).get("shards"));
assertThat(shards.size(), equalTo(2));
for (String shardId : shards.keySet()) {
@SuppressWarnings("unchecked") Map<String, ?> shardStoresStatus = (Map<String, ?>) shards.get(shardId);
assertThat(shardStoresStatus.containsKey("stores"), equalTo(true));
List<?> stores = (List<?>) shardStoresStatus.get("stores");
assertThat(stores.size(), equalTo(storeStatusList.size()));
for (int i = 0; i < stores.size(); i++) {
@SuppressWarnings("unchecked") Map<String, ?> storeInfo = ((Map<String, ?>) stores.get(i));
IndicesShardStoresResponse.StoreStatus storeStatus = storeStatusList.get(i);
assertThat(((String) storeInfo.get("allocation_id")), equalTo((storeStatus.getAllocationId())));
assertThat(storeInfo.containsKey("allocation"), equalTo(true));
assertThat(((String) storeInfo.get("allocation")), equalTo(storeStatus.getAllocationStatus().value()));
assertThat(storeInfo.containsKey(storeStatus.getNode().getId()), equalTo(true));
if (storeStatus.getStoreException() != null) {
assertThat(storeInfo.containsKey("store_exception"), equalTo(true));
}
}
}
}
}
}
use of org.opensearch.common.collect.ImmutableOpenIntMap in project OpenSearch by opensearch-project.
the class ClusterStateHealthTests method generateClusterStates.
private List<ClusterState> generateClusterStates(final ClusterState originalClusterState, final String indexName, final int numberOfReplicas, final boolean withPrimaryAllocationFailures) {
// generate random node ids
final Set<String> nodeIds = new HashSet<>();
final int numNodes = randomIntBetween(numberOfReplicas + 1, 10);
for (int i = 0; i < numNodes; i++) {
nodeIds.add(randomAlphaOfLength(8));
}
final List<ClusterState> clusterStates = new ArrayList<>();
clusterStates.add(originalClusterState);
ClusterState clusterState = originalClusterState;
// initialize primaries
RoutingTable routingTable = originalClusterState.routingTable();
IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary()) {
newIndexRoutingTable.addShard(shardRouting.initialize(randomFrom(nodeIds), null, shardRouting.getExpectedShardSize()));
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
clusterStates.add(clusterState);
// some primaries started
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
ImmutableOpenIntMap.Builder<Set<String>> allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() && randomBoolean()) {
final ShardRouting newShardRouting = shardRouting.moveToStarted();
allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
newIndexRoutingTable.addShard(newShardRouting);
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
IndexMetadata.Builder idxMetaBuilder = IndexMetadata.builder(clusterState.metadata().index(indexName));
for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
}
Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(idxMetaBuilder);
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metadata(metadataBuilder).build();
clusterStates.add(clusterState);
if (withPrimaryAllocationFailures) {
boolean alreadyFailedPrimary = false;
// some primaries failed to allocate
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() && (shardRouting.started() == false || alreadyFailedPrimary == false)) {
newIndexRoutingTable.addShard(shardRouting.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "unlucky shard")));
alreadyFailedPrimary = true;
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
return clusterStates;
}
// all primaries started
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() && shardRouting.started() == false) {
final ShardRouting newShardRouting = shardRouting.moveToStarted();
allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
newIndexRoutingTable.addShard(newShardRouting);
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
idxMetaBuilder = IndexMetadata.builder(clusterState.metadata().index(indexName));
for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
}
metadataBuilder = Metadata.builder(clusterState.metadata()).put(idxMetaBuilder);
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metadata(metadataBuilder).build();
clusterStates.add(clusterState);
// initialize replicas
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
Set<String> allocatedNodes = new HashSet<>();
allocatedNodes.add(primaryNodeId);
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() == false) {
// give the replica a different node id than the primary
String replicaNodeId = randomFrom(Sets.difference(nodeIds, allocatedNodes));
newIndexRoutingTable.addShard(shardRouting.initialize(replicaNodeId, null, shardRouting.getExpectedShardSize()));
allocatedNodes.add(replicaNodeId);
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
// some replicas started
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() == false && randomBoolean()) {
newIndexRoutingTable.addShard(shardRouting.moveToStarted());
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
// all replicas started
boolean replicaStateChanged = false;
indexRoutingTable = routingTable.index(indexName);
newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
if (shardRouting.primary() == false && shardRouting.started() == false) {
newIndexRoutingTable.addShard(shardRouting.moveToStarted());
replicaStateChanged = true;
} else {
newIndexRoutingTable.addShard(shardRouting);
}
}
}
// all of the replicas may have moved to started in the previous phase already
if (replicaStateChanged) {
routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
}
return clusterStates;
}
Aggregations