Examples with MetaData org.elasticsearch.cluster.metadata.MetaData used on opensource projects

Example 86 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class EnableAllocationTests method testClusterEnableOnlyPrimaries.

public void testClusterEnableOnlyPrimaries() {
    AllocationService strategy = createAllocationService(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.PRIMARIES.name()).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("--> adding two nodes do rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    routingTable = strategy.reroute(clusterState, "reroute").routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
    logger.info("--> start the shards (primaries)");
    routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
}

Example 87 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class RoutingNodesIntegrityTests method testBalanceAllNodesStarted.

public void testBalanceAllNodesStarted() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put("cluster.routing.allocation.node_initial_primaries_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1)).put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("test1")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    logger.info("Adding three node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    assertThat(assertShardStats(routingNodes), equalTo(true));
    // all shards are unassigned. so no inactive shards or primaries.
    assertThat(routingNodes.hasInactiveShards(), equalTo(false));
    assertThat(routingNodes.hasInactivePrimaries(), equalTo(false));
    assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(true));
    clusterState = strategy.reroute(clusterState, "reroute");
    routingNodes = clusterState.getRoutingNodes();
    assertThat(assertShardStats(routingNodes), equalTo(true));
    assertThat(routingNodes.hasInactiveShards(), equalTo(true));
    assertThat(routingNodes.hasInactivePrimaries(), equalTo(true));
    assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(false));
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    logger.info("Reroute, nothing should change");
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    routingNodes = clusterState.getRoutingNodes();
    assertThat(assertShardStats(routingNodes), equalTo(true));
    assertThat(routingNodes.hasInactiveShards(), equalTo(false));
    assertThat(routingNodes.hasInactivePrimaries(), equalTo(false));
    assertThat(routingNodes.hasUnassignedPrimaries(), equalTo(false));
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
}

Example 88 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class ShardsLimitAllocationTests method testClusterLevelShardsLimitAllocate.

public void testClusterLevelShardsLimitAllocate() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("Adding two nodes and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    logger.info("Start the primary shards");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
    assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2));
    // Bump the cluster total shards to 2
    strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 2).build());
    logger.info("Do another reroute, make sure shards are now allocated");
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
    assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
}

Example 89 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testSingleIndexStartedShard.

public void testSingleIndexStartedShard() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Adding one node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Rerouting again, nothing should change");
    clusterState = ClusterState.builder(clusterState).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    clusterState = newState;
    logger.info("Marking the shard as started");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Starting another node and making sure nothing changed");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Killing node1 where the shard is, checking the shard is unassigned");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
    newState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Bring node1 back, and see it's assinged");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node1"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Start another node, make sure that things remain the same (shard is in node2 and initializing)");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the shard on node 1");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
}

Example 90 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testMultiIndexEvenDistribution.

public void testMultiIndexEvenDistribution() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    final int numberOfIndices = 50;
    logger.info("Building initial routing table with " + numberOfIndices + " indices");
    MetaData.Builder metaDataBuilder = MetaData.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        metaDataBuilder.put(IndexMetaData.builder("test" + i).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
    }
    MetaData metaData = metaDataBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        routingTableBuilder.addAsNew(metaData.index("test" + i));
    }
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    assertThat(clusterState.routingTable().indicesRouting().size(), equalTo(numberOfIndices));
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Adding " + (numberOfIndices / 2) + " nodes");
    DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
    List<DiscoveryNode> nodes = new ArrayList<>();
    for (int i = 0; i < (numberOfIndices / 2); i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards initializing per node on the first 25 nodes
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    Set<String> encounteredIndices = new HashSet<>();
    for (RoutingNode routingNode : routingNodes) {
        assertThat(routingNode.numberOfShardsWithState(STARTED), equalTo(0));
        assertThat(routingNode.size(), equalTo(2));
        // make sure we still have 2 shards initializing per node on the only 25 nodes
        int nodeIndex = Integer.parseInt(routingNode.nodeId().substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
        // check that we don't have a shard associated with a node with the same index name (we have a single shard)
        for (ShardRouting shardRoutingEntry : routingNode) {
            assertThat(encounteredIndices, not(hasItem(shardRoutingEntry.getIndexName())));
            encounteredIndices.add(shardRoutingEntry.getIndexName());
        }
    }
    logger.info("Adding additional " + (numberOfIndices / 2) + " nodes, nothing should change");
    nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
    for (int i = (numberOfIndices / 2); i < numberOfIndices; i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Marking the shard as started");
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    int numberOfRelocatingShards = 0;
    int numberOfStartedShards = 0;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), anyOf(equalTo(STARTED), equalTo(RELOCATING)));
        if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == STARTED) {
            numberOfStartedShards++;
        } else if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == RELOCATING) {
            numberOfRelocatingShards++;
        }
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards either relocating or started on the first 25 nodes (still)
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    assertThat(numberOfRelocatingShards, equalTo(25));
    assertThat(numberOfStartedShards, equalTo(25));
}