Examples with ShardRouting org.opensearch.cluster.routing.ShardRouting used on opensource projects

Example 46 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class DiskThresholdDecider method sizeOfRelocatingShards.

/**
 * Returns the size of all shards that are currently being relocated to
 * the node, but may not be finished transferring yet.
 *
 * If subtractShardsMovingAway is true then the size of shards moving away is subtracted from the total size of all shards
 */
public static long sizeOfRelocatingShards(RoutingNode node, boolean subtractShardsMovingAway, String dataPath, ClusterInfo clusterInfo, Metadata metadata, RoutingTable routingTable) {
    // Account for reserved space wherever it is available
    final ClusterInfo.ReservedSpace reservedSpace = clusterInfo.getReservedSpace(node.nodeId(), dataPath);
    long totalSize = reservedSpace.getTotal();
    // NB this counts all shards on the node when the ClusterInfoService retrieved the node stats, which may include shards that are
    // no longer initializing because their recovery failed or was cancelled.
    // Where reserved space is unavailable (e.g. stats are out-of-sync) compute a conservative estimate for initialising shards
    final List<ShardRouting> initializingShards = node.shardsWithState(ShardRoutingState.INITIALIZING);
    initializingShards.removeIf(shardRouting -> reservedSpace.containsShardId(shardRouting.shardId()));
    for (ShardRouting routing : initializingShards) {
        if (routing.relocatingNodeId() == null) {
            // any additional space and can be ignored here
            continue;
        }
        final String actualPath = clusterInfo.getDataPath(routing);
        // free space
        if (actualPath == null || actualPath.equals(dataPath)) {
            totalSize += getExpectedShardSize(routing, 0L, clusterInfo, null, metadata, routingTable);
        }
    }
    if (subtractShardsMovingAway) {
        for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING)) {
            String actualPath = clusterInfo.getDataPath(routing);
            if (actualPath == null) {
                // we might know the path of this shard from before when it was relocating
                actualPath = clusterInfo.getDataPath(routing.cancelRelocation());
            }
            if (dataPath.equals(actualPath)) {
                totalSize -= getExpectedShardSize(routing, 0L, clusterInfo, null, metadata, routingTable);
            }
        }
    }
    return totalSize;
}

Example 47 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class ClusterInfoServiceIT method testClusterInfoServiceCollectsInformation.

public void testClusterInfoServiceCollectsInformation() {
    internalCluster().startNodes(2);
    final String indexName = randomBoolean() ? randomAlphaOfLength(5).toLowerCase(Locale.ROOT) : TEST_SYSTEM_INDEX_NAME;
    assertAcked(prepareCreate(indexName).setSettings(Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean()).build()));
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose(indexName));
    }
    ensureGreen(indexName);
    InternalTestCluster internalTestCluster = internalCluster();
    // Get the cluster info service on the master node
    final InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster.getInstance(ClusterInfoService.class, internalTestCluster.getMasterName());
    infoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
    ClusterInfo info = infoService.refresh();
    assertNotNull("info should not be null", info);
    ImmutableOpenMap<String, DiskUsage> leastUsages = info.getNodeLeastAvailableDiskUsages();
    ImmutableOpenMap<String, DiskUsage> mostUsages = info.getNodeMostAvailableDiskUsages();
    ImmutableOpenMap<String, Long> shardSizes = info.shardSizes;
    assertNotNull(leastUsages);
    assertNotNull(shardSizes);
    assertThat("some usages are populated", leastUsages.values().size(), Matchers.equalTo(2));
    assertThat("some shard sizes are populated", shardSizes.values().size(), greaterThan(0));
    for (ObjectCursor<DiskUsage> usage : leastUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<DiskUsage> usage : mostUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<Long> size : shardSizes.values()) {
        logger.info("--> shard size: {}", size.value);
        assertThat("shard size is greater than 0", size.value, greaterThanOrEqualTo(0L));
    }
    ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getMasterName());
    ClusterState state = clusterService.state();
    for (ShardRouting shard : state.routingTable().allShards()) {
        String dataPath = info.getDataPath(shard);
        assertNotNull(dataPath);
        String nodeId = shard.currentNodeId();
        DiscoveryNode discoveryNode = state.getNodes().get(nodeId);
        IndicesService indicesService = internalTestCluster.getInstance(IndicesService.class, discoveryNode.getName());
        IndexService indexService = indicesService.indexService(shard.index());
        IndexShard indexShard = indexService.getShardOrNull(shard.id());
        assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath);
        assertTrue(info.getReservedSpace(nodeId, dataPath).containsShardId(shard.shardId()));
    }
}

Example 48 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class ClusterStateDiffIT method randomChangeToIndexRoutingTable.

/**
 * Randomly updates index routing table in the cluster state
 */
private IndexRoutingTable randomChangeToIndexRoutingTable(IndexRoutingTable original, String[] nodes) {
    IndexRoutingTable.Builder builder = IndexRoutingTable.builder(original.getIndex());
    for (ObjectCursor<IndexShardRoutingTable> indexShardRoutingTable : original.shards().values()) {
        Set<String> availableNodes = Sets.newHashSet(nodes);
        for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
            availableNodes.remove(shardRouting.currentNodeId());
            if (shardRouting.relocating()) {
                availableNodes.remove(shardRouting.relocatingNodeId());
            }
        }
        for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
            final ShardRouting updatedShardRouting = randomChange(shardRouting, availableNodes);
            availableNodes.remove(updatedShardRouting.currentNodeId());
            if (shardRouting.relocating()) {
                availableNodes.remove(updatedShardRouting.relocatingNodeId());
            }
            builder.addShard(updatedShardRouting);
        }
    }
    return builder.build();
}

Example 49 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class ReplicaToPrimaryPromotionIT method testPromoteReplicaToPrimary.

public void testPromoteReplicaToPrimary() throws Exception {
    final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
    createIndex(indexName);
    final int numOfDocs = scaledRandomIntBetween(0, 200);
    if (numOfDocs > 0) {
        try (BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) {
            waitForDocs(numOfDocs, indexer);
        }
        refresh(indexName);
    }
    assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);
    ensureGreen(indexName);
    // sometimes test with a closed index
    final IndexMetadata.State indexState = randomFrom(IndexMetadata.State.OPEN, IndexMetadata.State.CLOSE);
    if (indexState == IndexMetadata.State.CLOSE) {
        CloseIndexResponse closeIndexResponse = client().admin().indices().prepareClose(indexName).get();
        assertThat("close index not acked - " + closeIndexResponse, closeIndexResponse.isAcknowledged(), equalTo(true));
        ensureGreen(indexName);
    }
    // pick up a data node that contains a random primary shard
    ClusterState state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState();
    final int numShards = state.metadata().index(indexName).getNumberOfShards();
    final ShardRouting primaryShard = state.routingTable().index(indexName).shard(randomIntBetween(0, numShards - 1)).primaryShard();
    final DiscoveryNode randomNode = state.nodes().resolveNode(primaryShard.currentNodeId());
    // stop the random data node, all remaining shards are promoted to primaries
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomNode.getName()));
    ensureYellowAndNoInitializingShards(indexName);
    state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState();
    for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(indexName)) {
        for (ShardRouting shardRouting : shardRoutingTable.activeShards()) {
            assertThat(shardRouting + " should be promoted as a primary", shardRouting.primary(), is(true));
        }
    }
    if (indexState == IndexMetadata.State.CLOSE) {
        assertAcked(client().admin().indices().prepareOpen(indexName));
        ensureYellowAndNoInitializingShards(indexName);
    }
    assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);
}

Example 50 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class IndicesStoreIntegrationIT method testShardActiveElseWhere.

public void testShardActiveElseWhere() throws Exception {
    List<String> nodes = internalCluster().startNodes(2);
    final String masterNode = internalCluster().getMasterName();
    final String nonMasterNode = nodes.get(0).equals(masterNode) ? nodes.get(1) : nodes.get(0);
    final String masterId = internalCluster().clusterService(masterNode).localNode().getId();
    final String nonMasterId = internalCluster().clusterService(nonMasterNode).localNode().getId();
    final int numShards = scaledRandomIntBetween(2, 10);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards)));
    ensureGreen("test");
    waitNoPendingTasksOnAll();
    ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
    final Index index = stateResponse.getState().metadata().index("test").getIndex();
    RoutingNode routingNode = stateResponse.getState().getRoutingNodes().node(nonMasterId);
    final int[] node2Shards = new int[routingNode.numberOfOwningShards()];
    int i = 0;
    for (ShardRouting shardRouting : routingNode) {
        node2Shards[i] = shardRouting.shardId().id();
        i++;
    }
    logger.info("Node [{}] has shards: {}", nonMasterNode, Arrays.toString(node2Shards));
    // disable relocations when we do this, to make sure the shards are not relocated from node2
    // due to rebalancing, and delete its content
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)).get();
    ClusterApplierService clusterApplierService = internalCluster().getInstance(ClusterService.class, nonMasterNode).getClusterApplierService();
    ClusterState currentState = clusterApplierService.state();
    IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
    for (int j = 0; j < numShards; j++) {
        indexRoutingTableBuilder.addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(index, j)).addShard(TestShardRouting.newShardRouting("test", j, masterId, true, ShardRoutingState.STARTED)).build());
    }
    ClusterState newState = ClusterState.builder(currentState).incrementVersion().routingTable(RoutingTable.builder().add(indexRoutingTableBuilder).build()).build();
    CountDownLatch latch = new CountDownLatch(1);
    clusterApplierService.onNewClusterState("test", () -> newState, new ClusterApplyListener() {

        @Override
        public void onSuccess(String source) {
            latch.countDown();
        }

        @Override
        public void onFailure(String source, Exception e) {
            latch.countDown();
            throw new AssertionError("Expected a proper response", e);
        }
    });
    latch.await();
    waitNoPendingTasksOnAll();
    logger.info("Checking if shards aren't removed");
    for (int shard : node2Shards) {
        assertShardExists(nonMasterNode, index, shard);
    }
}