Examples with BalancedShardsAllocator org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator used on opensource projects

Example 16 with BalancedShardsAllocator

use of org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator in project elasticsearch by elastic.

the class BalancedSingleShardTests method setupStateAndRebalance.

private Tuple<ClusterState, MoveDecision> setupStateAndRebalance(AllocationDecider allocationDecider, Settings balancerSettings, boolean rebalanceExpected) {
    AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {

        @Override
        public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
            return Decision.YES;
        }
    };
    List<AllocationDecider> allocationDeciders = Arrays.asList(rebalanceDecider, allocationDecider);
    final int numShards = randomIntBetween(8, 13);
    BalancedShardsAllocator allocator = new BalancedShardsAllocator(balancerSettings);
    ClusterState clusterState = ClusterStateCreationUtils.state("idx", 2, numShards);
    // add a new node so shards can be rebalanced there
    DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
    nodesBuilder.add(newNode(randomAsciiOfLength(7)));
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
    RoutingAllocation routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
    MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
    if (rebalanceExpected == false) {
        assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
    }
    return Tuple.tuple(clusterState, rebalanceDecision);
}

Example 17 with BalancedShardsAllocator

use of org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator in project elasticsearch by elastic.

the class BalancedSingleShardTests method testRebalancingNotAllowedDueToCanRebalance.

public void testRebalancingNotAllowedDueToCanRebalance() {
    final Decision canRebalanceDecision = randomFrom(Decision.NO, Decision.THROTTLE);
    AllocationDecider noRebalanceDecider = new AllocationDecider(Settings.EMPTY) {

        @Override
        public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
            return allocation.decision(canRebalanceDecision, "TEST", "foobar");
        }
    };
    BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
    ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED);
    ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
    RoutingAllocation routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(noRebalanceDecider)), clusterState);
    MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
    assertEquals(canRebalanceDecision.type(), rebalanceDecision.getClusterRebalanceDecision().type());
    assertEquals(AllocationDecision.fromDecisionType(canRebalanceDecision.type()), rebalanceDecision.getAllocationDecision());
    assertThat(rebalanceDecision.getExplanation(), containsString(canRebalanceDecision.type() == Type.THROTTLE ? "rebalancing is throttled" : "rebalancing is not allowed"));
    assertNotNull(rebalanceDecision.getNodeDecisions());
    assertNull(rebalanceDecision.getTargetNode());
    assertEquals(1, rebalanceDecision.getClusterRebalanceDecision().getDecisions().size());
    for (Decision subDecision : rebalanceDecision.getClusterRebalanceDecision().getDecisions()) {
        assertEquals("foobar", ((Decision.Single) subDecision).getExplanation());
    }
    assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
}

Example 18 with BalancedShardsAllocator

use of org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator in project elasticsearch by elastic.

the class BalancedSingleShardTests method testRebalanceNonStartedShardNotAllowed.

public void testRebalanceNonStartedShardNotAllowed() {
    BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
    ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING));
    ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
    MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState)).getMoveDecision();
    assertSame(MoveDecision.NOT_TAKEN, rebalanceDecision);
}

Example 19 with BalancedShardsAllocator

use of org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator in project elasticsearch by elastic.

the class BalancedSingleShardTests method testSingleShardBalanceProducesSameResultsAsBalanceStep.

public void testSingleShardBalanceProducesSameResultsAsBalanceStep() {
    final String[] indices = { "idx1", "idx2" };
    // Create a cluster state with 2 indices, each with 1 started primary shard, and only
    // one node initially so that all primary shards get allocated to the same node.  We are only
    // using 2 indices (i.e. 2 total primary shards) because if we have any more than 2 started shards
    // in the routing table, then we have no guarantees about the order in which the 3 or more shards
    // are selected to be rebalanced to the new node, and hence the node to which they are rebalanced
    // is not deterministic.  Using only two shards guarantees that only one of those two shards will
    // be rebalanced, and so we pick the one that was chosen to be rebalanced and execute the single-shard
    // rebalance step on it to make sure it gets assigned to the same node.
    ClusterState clusterState = ClusterStateCreationUtils.state(1, indices, 1);
    // add new nodes so one of the primaries can be rebalanced
    DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
    int numAddedNodes = randomIntBetween(1, 5);
    // randomly select a subset of the newly added nodes to set filter allocation on (but not all)
    int excludeNodesSize = randomIntBetween(0, numAddedNodes - 1);
    final Set<String> excludeNodes = new HashSet<>();
    for (int i = 0; i < numAddedNodes; i++) {
        DiscoveryNode discoveryNode = newNode(randomAsciiOfLength(7));
        nodesBuilder.add(discoveryNode);
        if (i < excludeNodesSize) {
            excludeNodes.add(discoveryNode.getId());
        }
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    AllocationDecider allocationDecider = new AllocationDecider(Settings.EMPTY) {

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            if (excludeNodes.contains(node.nodeId())) {
                return Decision.NO;
            }
            return Decision.YES;
        }
    };
    AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {

        @Override
        public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
            return Decision.YES;
        }
    };
    List<AllocationDecider> allocationDeciders = Arrays.asList(rebalanceDecider, allocationDecider);
    RoutingAllocation routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
    // allocate and get the node that is now relocating
    BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
    allocator.allocate(routingAllocation);
    ShardRouting shardToRebalance = null;
    for (RoutingNode routingNode : routingAllocation.routingNodes()) {
        List<ShardRouting> relocatingShards = routingNode.shardsWithState(ShardRoutingState.RELOCATING);
        if (relocatingShards.size() > 0) {
            shardToRebalance = randomFrom(relocatingShards);
            break;
        }
    }
    routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
    routingAllocation.debugDecision(true);
    ShardRouting shard = clusterState.getRoutingNodes().activePrimary(shardToRebalance.shardId());
    MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
    assertEquals(shardToRebalance.relocatingNodeId(), rebalanceDecision.getTargetNode().getId());
    // make sure all excluded nodes returned a NO decision
    for (NodeAllocationResult nodeResult : rebalanceDecision.getNodeDecisions()) {
        if (excludeNodes.contains(nodeResult.getNode().getId())) {
            assertEquals(Type.NO, nodeResult.getCanAllocateDecision().type());
        }
    }
}

Example 20 with BalancedShardsAllocator

use of org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator in project elasticsearch by elastic.

the class MetaDataCreateIndexServiceTests method testValidateShrinkIndex.

public void testValidateShrinkIndex() {
    int numShards = randomIntBetween(2, 42);
    ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build());
    assertEquals("index [source] already exists", expectThrows(ResourceAlreadyExistsException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(state, "target", Collections.emptySet(), "source", Settings.EMPTY)).getMessage());
    assertEquals("no such index", expectThrows(IndexNotFoundException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY)).getMessage());
    assertEquals("can't shrink an index with only one shard", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", Settings.EMPTY)).getMessage());
    assertEquals("the number of target shards must be less that the number of source shards", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 10).build())).getMessage());
    assertEquals("index source must be read-only to shrink index. use \"index.blocks.write=true\"", expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY), "source", Collections.emptySet(), "target", Settings.EMPTY)).getMessage());
    assertEquals("index source must have all shards allocated on the same node to shrink index", expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", Settings.EMPTY)).getMessage());
    assertEquals("the number of source shards [8] must be a must be a multiple of [3]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 8, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 3).build())).getMessage());
    assertEquals("mappings are not allowed when shrinking indices, all mappings are copied from the source index", expectThrows(IllegalArgumentException.class, () -> {
        MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.singleton("foo"), "target", Settings.EMPTY);
    }).getMessage());
    // create one that won't fail
    ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
    RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    // now we start the shard
    routingTable = service.applyStartedShards(clusterState, routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    int targetShards;
    do {
        targetShards = randomIntBetween(1, numShards / 2);
    } while (isShrinkable(numShards, targetShards) == false);
    MetaDataCreateIndexService.validateShrinkIndex(clusterState, "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", targetShards).build());
}