Examples with ClusterSettings org.elasticsearch.common.settings.ClusterSettings used on opensource projects

Example 71 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project elasticsearch by elastic.

the class DiskThresholdDeciderUnitTests method testCanAllocateUsesMaxAvailableSpace.

public void testCanAllocateUsesMaxAvailableSpace() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    final Index index = metaData.index("test").getIndex();
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node_0).add(node_1)).build();
    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    // all full
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0));
    // all full
    leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, 0));
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    // 20 - 99 percent since after allocation there must be at least 10% left and shard is 10byte
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, randomIntBetween(20, 100)));
    // this is weird and smells like a bug! it should be up to 20%?
    mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, randomIntBetween(0, 10)));
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // 10 bytes
    shardSizes.put("[test][0][p]", 10L);
    final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
    allocation.debugDecision(true);
    Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(mostAvailableUsage.toString(), Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("enough disk for shard on node"));
    decision = decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation);
    assertEquals(mostAvailableUsage.toString(), Decision.Type.NO, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%], using more " + "disk space than the maximum allowed [90.0%]"));
}

Example 72 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project elasticsearch by elastic.

the class EnableAllocationTests method testEnableClusterBalance.

public void testEnableClusterBalance() {
    final boolean useClusterSetting = randomBoolean();
    final Rebalance allowedOnes = RandomPicks.randomFrom(random(), EnumSet.of(Rebalance.PRIMARIES, Rebalance.REPLICAS, Rebalance.ALL));
    Settings build = Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), // index settings override cluster settings
    useClusterSetting ? Rebalance.NONE : RandomPicks.randomFrom(random(), Rebalance.values())).put(ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.getKey(), 3).put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 10).build();
    ClusterSettings clusterSettings = new ClusterSettings(build, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    AllocationService strategy = createAllocationService(build, clusterSettings, random());
    Settings indexSettings = useClusterSetting ? Settings.EMPTY : Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE).build();
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(indexSettings)).numberOfShards(3).numberOfReplicas(1)).put(IndexMetaData.builder("always_disabled").settings(settings(Version.CURRENT).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("always_disabled")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    logger.info("--> adding one nodes and do rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
    logger.info("--> start the shards (primaries)");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
    logger.info("--> adding one nodes and do rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
    assertThat(clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(0));
    if (useClusterSetting) {
        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData()).transientSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), allowedOnes).build())).build();
    } else {
        IndexMetaData meta = clusterState.getMetaData().index("test");
        IndexMetaData meta1 = clusterState.getMetaData().index("always_disabled");
        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData()).removeAllIndices().put(IndexMetaData.builder(meta1)).put(IndexMetaData.builder(meta).settings(Settings.builder().put(meta.getSettings()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), allowedOnes).build()))).build();
    }
    clusterSettings.applySettings(clusterState.metaData().settings());
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat("expected 6 shards to be started 2 to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6));
    assertThat("expected 2 shards to relocate useClusterSettings: " + useClusterSetting, clusterState.getRoutingNodes().shardsWithState(RELOCATING).size(), equalTo(2));
    List<ShardRouting> mutableShardRoutings = clusterState.getRoutingNodes().shardsWithState(RELOCATING);
    switch(allowedOnes) {
        case PRIMARIES:
            for (ShardRouting routing : mutableShardRoutings) {
                assertTrue("only primaries are allowed to relocate", routing.primary());
                assertThat("only test index can rebalance", routing.getIndexName(), equalTo("test"));
            }
            break;
        case REPLICAS:
            for (ShardRouting routing : mutableShardRoutings) {
                assertFalse("only replicas are allowed to relocate", routing.primary());
                assertThat("only test index can rebalance", routing.getIndexName(), equalTo("test"));
            }
            break;
        case ALL:
            for (ShardRouting routing : mutableShardRoutings) {
                assertThat("only test index can rebalance", routing.getIndexName(), equalTo("test"));
            }
            break;
        default:
            fail("only replicas, primaries or all are allowed");
    }
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
}

Example 73 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project elasticsearch by elastic.

the class RoutingIteratorTests method testShardsAndPreferNodeRouting.

public void testShardsAndPreferNodeRouting() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).localNodeId("node1")).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(1));
    //check node preference, first without preference to see they switch
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    String firstRoundNodeId = shardIterators.iterator().next().nextOrNull().currentNodeId();
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId)));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1,node2");
    assertThat(shardIterators.size(), equalTo(1));
    Iterator<ShardIterator> iterator = shardIterators.iterator();
    final ShardIterator it = iterator.next();
    assertThat(it.shardId().id(), equalTo(0));
    final String firstNodeId = it.nextOrNull().currentNodeId();
    assertThat(firstNodeId, anyOf(equalTo("node1"), equalTo("node2")));
    if ("node1".equals(firstNodeId)) {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node2"));
    } else {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node1"));
    }
}

Example 74 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project elasticsearch by elastic.

the class BigArraysTests method testMaxSizeExceededOnNew.

public void testMaxSizeExceededOnNew() throws Exception {
    final int size = scaledRandomIntBetween(5, 1 << 22);
    for (String type : Arrays.asList("Byte", "Int", "Long", "Float", "Double", "Object")) {
        HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(Settings.builder().put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), size - 1, ByteSizeUnit.BYTES).build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
        BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
        Method create = BigArrays.class.getMethod("new" + type + "Array", long.class);
        try {
            create.invoke(bigArrays, size);
            fail("expected an exception on " + create);
        } catch (InvocationTargetException e) {
            assertTrue(e.getCause() instanceof CircuitBreakingException);
        }
        assertEquals(0, hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
    }
}

Example 75 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project elasticsearch by elastic.

the class BigArraysTests method testMaxSizeExceededOnResize.

public void testMaxSizeExceededOnResize() throws Exception {
    for (String type : Arrays.asList("Byte", "Int", "Long", "Float", "Double", "Object")) {
        final long maxSize = randomIntBetween(1 << 10, 1 << 22);
        HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService(Settings.builder().put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), maxSize, ByteSizeUnit.BYTES).build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
        BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
        Method create = BigArrays.class.getMethod("new" + type + "Array", long.class);
        final int size = scaledRandomIntBetween(1, 20);
        BigArray array = (BigArray) create.invoke(bigArrays, size);
        Method resize = BigArrays.class.getMethod("resize", array.getClass().getInterfaces()[0], long.class);
        while (true) {
            long newSize = array.size() * 2;
            try {
                array = (BigArray) resize.invoke(bigArrays, array, newSize);
            } catch (InvocationTargetException e) {
                assertTrue(e.getCause() instanceof CircuitBreakingException);
                break;
            }
        }
        assertEquals(array.ramBytesUsed(), hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
        array.close();
        assertEquals(0, hcbs.getBreaker(CircuitBreaker.REQUEST).getUsed());
    }
}