Examples with ClusterSettings org.elasticsearch.common.settings.ClusterSettings used on opensource projects

Example 36 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class DiskThresholdDeciderTests method testDiskThresholdWithShardSizes.

@Test
public void testDiskThresholdWithShardSizes() {
    Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "false").put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "71%").build();
    ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 69% used
    usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 31));
    // 99% used
    usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1));
    ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    ImmutableOpenMap.Builder<String, Long> shardSizesBuilder = ImmutableOpenMap.builder();
    // 10 bytes
    shardSizesBuilder.put("[test][0][p]", 10L);
    ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
    final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
    AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), makeDecider(diskSettings))));
    ClusterInfoService cis = () -> {
        logger.info("--> calling fake getClusterInfo");
        return clusterInfo;
    };
    AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "false").build())).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).build();
    logger.info("--> adding node1");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    routingTable = strategy.reroute(clusterState, "reroute").routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    logger.info("--> start the shards (primaries)");
    routingTable = startInitializingShardsAndReroute(strategy, clusterState).routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    logShardStates(clusterState);
    // Shard can't be allocated to node1 (or node2) because it would cause too much usage
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
    // No shards are started, no nodes have enough disk for allocation
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
}

Example 37 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class DiskThresholdDeciderTests method testDiskThreshold.

@Test
public void testDiskThreshold() {
    Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8).build();
    ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 90% used
    usagesBuilder.put("node1", new DiskUsage("node1", "node1", "/dev/null", 100, 10));
    // 65% used
    usagesBuilder.put("node2", new DiskUsage("node2", "node2", "/dev/null", 100, 35));
    // 40% used
    usagesBuilder.put("node3", new DiskUsage("node3", "node3", "/dev/null", 100, 60));
    // 20% used
    usagesBuilder.put("node4", new DiskUsage("node4", "node4", "/dev/null", 100, 80));
    ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    ImmutableOpenMap.Builder<String, Long> shardSizesBuilder = ImmutableOpenMap.builder();
    // 10 bytes
    shardSizesBuilder.put("[test][0][p]", 10L);
    shardSizesBuilder.put("[test][0][r]", 10L);
    ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
    final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings))));
    ClusterInfoService cis = () -> {
        logger.info("--> calling fake getClusterInfo");
        return clusterInfo;
    };
    AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "false"))).build();
    final RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding two nodes");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // Primary shard should be initializing, replica should not
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    // Assert that we're able to start the primary
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
    // Assert that node1 didn't get any shards because its disk usage is too high
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
    logger.info("--> start the shards (replicas)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    // Assert that the replica couldn't be started since node1 doesn't have enough space
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
    logger.info("--> adding node3");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // Assert that the replica is initialized now that node3 is available with enough space
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
    logger.info("--> start the shards (replicas)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    // Assert that the replica couldn't be started since node1 doesn't have enough space
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
    logger.info("--> changing decider settings");
    // Set the low threshold to 60 instead of 70
    // Set the high threshold to 70 instead of 80
    // node2 now should not have new shards allocated to it, but shards can remain
    diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%").put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7).build();
    deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings))));
    strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // Shards remain started
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
    logger.info("--> changing settings again");
    // Set the low threshold to 50 instead of 60
    // Set the high threshold to 60 instead of 70
    // node2 now should not have new shards allocated to it, and shards cannot remain
    diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.5).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6).build();
    deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings))));
    strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // Shards remain started
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
    // Shard hasn't been moved off of node2 yet because there's nowhere for it to go
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
    logger.info("--> adding node4");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logShardStates(clusterState);
    // Shards remain started
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
    logger.info("--> apply INITIALIZING shards");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logShardStates(clusterState);
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
    // Node4 is available now, so the shard is moved off of node2
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
    assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node4").size(), equalTo(1));
}

Example 38 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class DiskThresholdDeciderUnitTests method testCannotAllocateDueToLackOfDiskResources.

@Test
public void testCannotAllocateDueToLackOfDiskResources() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    final Index index = metadata.index("test").getIndex();
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node_0).add(node_1)).build();
    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    // all full
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0));
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    final int freeBytes = randomIntBetween(20, 100);
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, freeBytes));
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // way bigger than available space
    final long shardSize = randomIntBetween(110, 1000);
    shardSizes.put("[test][0][p]", shardSize);
    ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
    allocation.debugDecision(true);
    Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(Decision.Type.NO, decision.type());
    assertThat(decision.getExplanation(), containsString("allocating the shard to this node will bring the node above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=90%] " + "and cause it to have less than the minimum required [0b] of free space " + "(free: [" + freeBytes + "b], estimated shard size: [" + shardSize + "b])"));
}

Example 39 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class ReconfiguratorTests method testDynamicSetting.

public void testDynamicSetting() {
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    final Reconfigurator reconfigurator = new Reconfigurator(Settings.EMPTY, clusterSettings);
    final VotingConfiguration initialConfig = conf("a", "b", "c", "d", "e");
    Set<DiscoveryNode> twoNodes = nodes("a", "b");
    Set<DiscoveryNode> threeNodes = nodes("a", "b", "c");
    // default is "true"
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c")));
    // update to "false"
    clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "false").build());
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), // no quorum
    sameInstance(initialConfig));
    assertThat(reconfigurator.reconfigure(threeNodes, retired(), randomFrom(threeNodes), initialConfig), equalTo(conf("a", "b", "c", "d", "e")));
    assertThat(reconfigurator.reconfigure(threeNodes, retired("d"), randomFrom(threeNodes), initialConfig), equalTo(conf("a", "b", "c", "e")));
    // explicitly set to "true"
    clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "true").build());
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c")));
    expectThrows(IllegalArgumentException.class, () -> clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "blah").build()));
}

Example 40 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class NodeJoinTests method setupMasterServiceAndCoordinator.

private void setupMasterServiceAndCoordinator(long term, ClusterState initialState, MasterService masterService, ThreadPool threadPool, Random random) {
    if (this.masterService != null || coordinator != null) {
        throw new IllegalStateException("method setupMasterServiceAndCoordinator can only be called once");
    }
    this.masterService = masterService;
    CapturingTransport capturingTransport = new CapturingTransport() {

        @Override
        protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode destination) {
            if (action.equals(HANDSHAKE_ACTION_NAME)) {
                handleResponse(requestId, new TransportService.HandshakeResponse(destination, initialState.getClusterName(), destination.getVersion()));
            } else if (action.equals(JoinHelper.VALIDATE_JOIN_ACTION_NAME)) {
                handleResponse(requestId, new TransportResponse.Empty());
            } else {
                super.onSendRequest(requestId, action, request, destination);
            }
        }
    };
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    TransportService transportService = capturingTransport.createTransportService(Settings.EMPTY, threadPool, x -> initialState.nodes().getLocalNode(), clusterSettings);
    coordinator = new Coordinator("test_node", Settings.EMPTY, clusterSettings, transportService, writableRegistry(), ESAllocationTestCase.createAllocationService(Settings.EMPTY), masterService, () -> new InMemoryPersistedState(term, initialState), r -> emptyList(), new NoOpClusterApplier(), Collections.emptyList(), random, (s, p, r) -> {
    });
    transportService.start();
    transportService.acceptIncomingRequests();
    transport = capturingTransport;
    coordinator.start();
    coordinator.startInitialJoin();
}