Examples with MockInternalClusterInfoService org.opensearch.cluster.MockInternalClusterInfoService used on opensource projects

Example 1 with MockInternalClusterInfoService

use of org.opensearch.cluster.MockInternalClusterInfoService in project OpenSearch by opensearch-project.

the class MockDiskUsagesIT method testDoesNotExceedLowWatermarkWhenRebalancing.

public void testDoesNotExceedLowWatermarkWhenRebalancing() throws Exception {
    for (int i = 0; i < 3; i++) {
        // ensure that each node has a single data path
        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()));
    }
    final AtomicReference<ClusterState> clusterManagerAppliedClusterState = new AtomicReference<>();
    final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService();
    final List<String> nodeIds = StreamSupport.stream(client().admin().cluster().prepareState().get().getState().getRoutingNodes().spliterator(), false).map(RoutingNode::nodeId).collect(Collectors.toList());
    internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> {
        assertThat(event.state().getRoutingNodes().node(nodeIds.get(2)).size(), lessThanOrEqualTo(1));
        clusterManagerAppliedClusterState.set(event.state());
        // so that a subsequent reroute sees disk usage according to the current state
        clusterInfoService.refresh();
    });
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "85%").put(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "100%").put(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "100%")));
    // shards are 1 byte large
    clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 1L);
    // node 2 only has space for one shard
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L, discoveryNode.getId().equals(nodeIds.get(2)) ? 150L - clusterManagerAppliedClusterState.get().getRoutingNodes().node(nodeIds.get(2)).numberOfOwningShards() : 1000L));
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0).put(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_id").getKey(), nodeIds.get(2))));
    ensureGreen("test");
    assertBusy(() -> {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has 3 shards", shardCountByNodeId.get(nodeIds.get(0)), equalTo(3));
        assertThat("node1 has 3 shards", shardCountByNodeId.get(nodeIds.get(1)), equalTo(3));
        assertThat("node2 has 0 shards", shardCountByNodeId.get(nodeIds.get(2)), equalTo(0));
    });
    assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_id").getKey())));
    logger.info("--> waiting for shards to relocate onto node [{}]", nodeIds.get(2));
    ensureGreen("test");
    assertThat("node2 has 1 shard", getShardCountByNodeId().get(nodeIds.get(2)), equalTo(1));
}

Example 2 with MockInternalClusterInfoService

use of org.opensearch.cluster.MockInternalClusterInfoService in project OpenSearch by opensearch-project.

the class MockDiskUsagesIT method testAutomaticReleaseOfIndexBlock.

public void testAutomaticReleaseOfIndexBlock() throws Exception {
    for (int i = 0; i < 3; i++) {
        // ensure that each node has a single data path
        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()));
    }
    final List<String> nodeIds = StreamSupport.stream(client().admin().cluster().prepareState().get().getState().getRoutingNodes().spliterator(), false).map(RoutingNode::nodeId).collect(Collectors.toList());
    final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService();
    clusterInfoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
    // prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
    clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
    // start with all nodes below the low watermark
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(15, 100)));
    // we have to consistently use bytes or percentage for the disk watermark settings
    final boolean watermarkBytes = randomBoolean();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%").put(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%").put(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), watermarkBytes ? "5b" : "95%").put(CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "150ms")));
    // Create an index with 6 shards so we can check allocation for it
    prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)).get();
    ensureGreen("test");
    {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has 2 shards", shardCountByNodeId.get(nodeIds.get(0)), equalTo(2));
        assertThat("node1 has 2 shards", shardCountByNodeId.get(nodeIds.get(1)), equalTo(2));
        assertThat("node2 has 2 shards", shardCountByNodeId.get(nodeIds.get(2)), equalTo(2));
    }
    client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
    assertSearchHits(client().prepareSearch("test").get(), "1");
    // Move all nodes above the low watermark so no shard movement can occur, and at least one node above the flood stage watermark so
    // the index is blocked
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 4) : between(0, 9)));
    assertBusy(() -> assertBlocked(client().prepareIndex().setIndex("test").setId("1").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK));
    assertFalse(client().admin().cluster().prepareHealth("test").setWaitForEvents(Priority.LANGUID).get().isTimedOut());
    // Cannot add further documents
    assertBlocked(client().prepareIndex().setIndex("test").setId("2").setSource("foo", "bar"), IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
    assertSearchHits(client().prepareSearch("test").get(), "1");
    logger.info("--> index is confirmed read-only, releasing disk space");
    // Move all nodes below the high watermark so that the index is unblocked
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
    // Attempt to create a new document until DiskUsageMonitor unblocks the index
    assertBusy(() -> {
        try {
            client().prepareIndex("test").setId("3").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
        } catch (ClusterBlockException e) {
            throw new AssertionError("retrying", e);
        }
    });
    assertSearchHits(client().prepareSearch("test").get(), "1", "3");
}

Example 3 with MockInternalClusterInfoService

use of org.opensearch.cluster.MockInternalClusterInfoService in project OpenSearch by opensearch-project.

the class MockNode method newClusterInfoService.

@Override
protected ClusterInfoService newClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) {
    if (getPluginsService().filterPlugins(MockInternalClusterInfoService.TestPlugin.class).isEmpty()) {
        return super.newClusterInfoService(settings, clusterService, threadPool, client);
    } else {
        final MockInternalClusterInfoService service = new MockInternalClusterInfoService(settings, clusterService, threadPool, client);
        clusterService.addListener(service);
        return service;
    }
}

Example 4 with MockInternalClusterInfoService

use of org.opensearch.cluster.MockInternalClusterInfoService in project OpenSearch by opensearch-project.

the class MockDiskUsagesIT method testRerouteOccursOnDiskPassingHighWatermark.

public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
    for (int i = 0; i < 3; i++) {
        // ensure that each node has a single data path
        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()));
    }
    final List<String> nodeIds = StreamSupport.stream(client().admin().cluster().prepareState().get().getState().getRoutingNodes().spliterator(), false).map(RoutingNode::nodeId).collect(Collectors.toList());
    final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService();
    clusterInfoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
    // prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
    clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
    // start with all nodes below the watermark
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
    // we have to consistently use bytes or percentage for the disk watermark settings
    final boolean watermarkBytes = randomBoolean();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%").put(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%").put(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), watermarkBytes ? "0b" : "100%").put(CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms")));
    // Create an index with 10 shards so we can check allocation for it
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 10).put("number_of_replicas", 0)));
    ensureGreen("test");
    assertBusy(() -> {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(0)), greaterThanOrEqualTo(3));
        assertThat("node1 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(1)), greaterThanOrEqualTo(3));
        assertThat("node2 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(2)), greaterThanOrEqualTo(3));
    });
    // move node2 above high watermark
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 9) : between(10, 100)));
    logger.info("--> waiting for shards to relocate off node [{}]", nodeIds.get(2));
    assertBusy(() -> {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has 5 shards", shardCountByNodeId.get(nodeIds.get(0)), equalTo(5));
        assertThat("node1 has 5 shards", shardCountByNodeId.get(nodeIds.get(1)), equalTo(5));
        assertThat("node2 has 0 shards", shardCountByNodeId.get(nodeIds.get(2)), equalTo(0));
    });
    // move all nodes below watermark again
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
    logger.info("--> waiting for shards to rebalance back onto node [{}]", nodeIds.get(2));
    assertBusy(() -> {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(0)), greaterThanOrEqualTo(3));
        assertThat("node1 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(1)), greaterThanOrEqualTo(3));
        assertThat("node2 has at least 3 shards", shardCountByNodeId.get(nodeIds.get(2)), greaterThanOrEqualTo(3));
    });
}

Example 5 with MockInternalClusterInfoService

use of org.opensearch.cluster.MockInternalClusterInfoService in project OpenSearch by opensearch-project.

the class MockDiskUsagesIT method testMovesShardsOffSpecificDataPathAboveWatermark.

public void testMovesShardsOffSpecificDataPathAboveWatermark() throws Exception {
    // start one node with two data paths
    final Path pathOverWatermark = createTempDir();
    final Settings.Builder twoPathSettings = Settings.builder();
    if (randomBoolean()) {
        twoPathSettings.putList(Environment.PATH_DATA_SETTING.getKey(), createTempDir().toString(), pathOverWatermark.toString());
    } else {
        twoPathSettings.putList(Environment.PATH_DATA_SETTING.getKey(), pathOverWatermark.toString(), createTempDir().toString());
    }
    internalCluster().startNode(twoPathSettings);
    final String nodeWithTwoPaths = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0).getNode().getId();
    // other two nodes have one data path each
    internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()));
    internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()));
    final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService();
    // prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
    clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
    // start with all paths below the watermark
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%").put(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "90%").put(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "100%")));
    final List<String> nodeIds = StreamSupport.stream(client().admin().cluster().prepareState().get().getState().getRoutingNodes().spliterator(), false).map(RoutingNode::nodeId).collect(Collectors.toList());
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
    ensureGreen("test");
    {
        final Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
        assertThat("node0 has 2 shards", shardCountByNodeId.get(nodeIds.get(0)), equalTo(2));
        assertThat("node1 has 2 shards", shardCountByNodeId.get(nodeIds.get(1)), equalTo(2));
        assertThat("node2 has 2 shards", shardCountByNodeId.get(nodeIds.get(2)), equalTo(2));
    }
    final long shardsOnGoodPath = Arrays.stream(client().admin().indices().prepareStats("test").get().getShards()).filter(shardStats -> shardStats.getShardRouting().currentNodeId().equals(nodeWithTwoPaths) && shardStats.getDataPath().startsWith(pathOverWatermark.toString()) == false).count();
    logger.info("--> shards on good path: [{}]", shardsOnGoodPath);
    // disable rebalancing, or else we might move shards back onto the over-full path since we're not faking that
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
    // one of the paths on node0 suddenly exceeds the high watermark
    clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100L, fsInfoPath.getPath().startsWith(pathOverWatermark.toString()) ? between(0, 9) : between(10, 100)));
    logger.info("--> waiting for shards to relocate off path [{}]", pathOverWatermark);
    assertBusy(() -> {
        for (final ShardStats shardStats : client().admin().indices().prepareStats("test").get().getShards()) {
            assertThat(shardStats.getDataPath(), not(startsWith(pathOverWatermark.toString())));
        }
    });
    ensureGreen("test");
    for (final ShardStats shardStats : client().admin().indices().prepareStats("test").get().getShards()) {
        assertThat(shardStats.getDataPath(), not(startsWith(pathOverWatermark.toString())));
    }
    assertThat("should not have moved any shards off of the path that wasn't too full", Arrays.stream(client().admin().indices().prepareStats("test").get().getShards()).filter(shardStats -> shardStats.getShardRouting().currentNodeId().equals(nodeWithTwoPaths) && shardStats.getDataPath().startsWith(pathOverWatermark.toString()) == false).count(), equalTo(shardsOnGoodPath));
}