Examples with ShardRouting org.elasticsearch.cluster.routing.ShardRouting used on opensource projects

Example 51 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class AwarenessAllocationIT method testAwarenessZones.

public void testAwarenessZones() throws Exception {
    Settings commonSettings = Settings.builder().put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a,b").put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone").put(ZenDiscovery.JOIN_TIMEOUT_SETTING.getKey(), "10s").build();
    logger.info("--> starting 4 nodes on different zones");
    List<String> nodes = internalCluster().startNodes(Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "b").build(), Settings.builder().put(commonSettings).put("node.attr.zone", "a").build());
    String A_0 = nodes.get(0);
    String B_0 = nodes.get(1);
    String B_1 = nodes.get(2);
    String A_1 = nodes.get(3);
    logger.info("--> waiting for nodes to form a cluster");
    ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("4").execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 5).put("index.number_of_replicas", 1)).execute().actionGet();
    logger.info("--> waiting for shards to be allocated");
    health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNoRelocatingShards(true).execute().actionGet();
    assertThat(health.isTimedOut(), equalTo(false));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
            }
        }
    }
    assertThat(counts.get(A_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_1), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(A_0), anyOf(equalTo(2), equalTo(3)));
    assertThat(counts.get(B_0), anyOf(equalTo(2), equalTo(3)));
}

Example 52 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class ClusterRerouteIT method testClusterRerouteWithBlocks.

public void testClusterRerouteWithBlocks() throws Exception {
    List<String> nodesIds = internalCluster().startNodes(2);
    logger.info("--> create an index with 1 shard and 0 replicas");
    assertAcked(prepareCreate("test-blocks").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)));
    ensureGreen("test-blocks");
    logger.info("--> check that the index has 1 shard");
    ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
    List<ShardRouting> shards = state.routingTable().allShards("test-blocks");
    assertThat(shards, hasSize(1));
    logger.info("--> check that the shard is allocated");
    ShardRouting shard = shards.get(0);
    assertThat(shard.assignedToNode(), equalTo(true));
    logger.info("--> retrieve the node where the shard is allocated");
    DiscoveryNode node = state.nodes().resolveNode(shard.currentNodeId());
    assertNotNull(node);
    // toggle is used to mve the shard from one node to another
    int toggle = nodesIds.indexOf(node.getName());
    // Rerouting shards is not blocked
    for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
        try {
            enableIndexBlock("test-blocks", blockSetting);
            assertAcked(client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test-blocks", 0, nodesIds.get(toggle % 2), nodesIds.get(++toggle % 2))));
            ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForYellowStatus().setWaitForNoRelocatingShards(true).execute().actionGet();
            assertThat(healthResponse.isTimedOut(), equalTo(false));
        } finally {
            disableIndexBlock("test-blocks", blockSetting);
        }
    }
    // Rerouting shards is blocked when the cluster is read only
    try {
        setClusterReadOnly(true);
        assertBlocked(client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test-blocks", 1, nodesIds.get(toggle % 2), nodesIds.get(++toggle % 2))));
    } finally {
        setClusterReadOnly(false);
    }
}

Example 53 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class FilteringAllocationIT method testDisablingAllocationFiltering.

public void testDisablingAllocationFiltering() throws Exception {
    logger.info("--> starting 2 nodes");
    List<String> nodesIds = internalCluster().startNodes(2);
    final String node_0 = nodesIds.get(0);
    final String node_1 = nodesIds.get(1);
    assertThat(cluster().size(), equalTo(2));
    logger.info("--> creating an index with no replicas");
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)).execute().actionGet();
    ensureGreen();
    logger.info("--> index some data");
    for (int i = 0; i < 100; i++) {
        client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
    }
    client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(100L));
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    IndexRoutingTable indexRoutingTable = clusterState.routingTable().index("test");
    int numShardsOnNode1 = 0;
    for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
        for (ShardRouting shardRouting : indexShardRoutingTable) {
            if ("node1".equals(clusterState.nodes().get(shardRouting.currentNodeId()).getName())) {
                numShardsOnNode1++;
            }
        }
    }
    if (numShardsOnNode1 > ThrottlingAllocationDecider.DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES) {
        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", numShardsOnNode1)).execute().actionGet();
    // make sure we can recover all the nodes at once otherwise we might run into a state where one of the shards has not yet started relocating
    // but we already fired up the request to wait for 0 relocating shards.
    }
    logger.info("--> remove index from the first node");
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.routing.allocation.exclude._name", node_0)).execute().actionGet();
    client().admin().cluster().prepareReroute().get();
    ensureGreen();
    logger.info("--> verify all shards are allocated on node_1 now");
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    indexRoutingTable = clusterState.routingTable().index("test");
    for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
        for (ShardRouting shardRouting : indexShardRoutingTable) {
            assertThat(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(node_1));
        }
    }
    logger.info("--> disable allocation filtering ");
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("index.routing.allocation.exclude._name", "")).execute().actionGet();
    client().admin().cluster().prepareReroute().get();
    ensureGreen();
    logger.info("--> verify that there are shards allocated on both nodes now");
    clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    assertThat(clusterState.routingTable().index("test").numberOfNodesShardsAreAllocatedOn(), equalTo(2));
}

Example 54 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class FilteringAllocationIT method testDecommissionNodeNoReplicas.

public void testDecommissionNodeNoReplicas() throws Exception {
    logger.info("--> starting 2 nodes");
    List<String> nodesIds = internalCluster().startNodes(2);
    final String node_0 = nodesIds.get(0);
    final String node_1 = nodesIds.get(1);
    assertThat(cluster().size(), equalTo(2));
    logger.info("--> creating an index with no replicas");
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)).execute().actionGet();
    ensureGreen();
    logger.info("--> index some data");
    for (int i = 0; i < 100; i++) {
        client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
    }
    client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> decommission the second node");
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.exclude._name", node_1)).execute().actionGet();
    waitForRelocation();
    logger.info("--> verify all are allocated on node1 now");
    ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
    for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            for (ShardRouting shardRouting : indexShardRoutingTable) {
                assertThat(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(node_0));
            }
        }
    }
    client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(100L));
}

Example 55 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class ClusterStateHealthTests method generateClusterStates.

private List<ClusterState> generateClusterStates(final ClusterState originalClusterState, final String indexName, final int numberOfReplicas, final boolean withPrimaryAllocationFailures) {
    // generate random node ids
    final Set<String> nodeIds = new HashSet<>();
    final int numNodes = randomIntBetween(numberOfReplicas + 1, 10);
    for (int i = 0; i < numNodes; i++) {
        nodeIds.add(randomAsciiOfLength(8));
    }
    final List<ClusterState> clusterStates = new ArrayList<>();
    clusterStates.add(originalClusterState);
    ClusterState clusterState = originalClusterState;
    // initialize primaries
    RoutingTable routingTable = originalClusterState.routingTable();
    IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
    IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary()) {
                newIndexRoutingTable.addShard(shardRouting.initialize(randomFrom(nodeIds), null, shardRouting.getExpectedShardSize()));
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    clusterStates.add(clusterState);
    // some primaries started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    ImmutableOpenIntMap.Builder<Set<String>> allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() && randomBoolean()) {
                final ShardRouting newShardRouting = shardRouting.moveToStarted();
                allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
                newIndexRoutingTable.addShard(newShardRouting);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    IndexMetaData.Builder idxMetaBuilder = IndexMetaData.builder(clusterState.metaData().index(indexName));
    for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
        idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
    }
    MetaData.Builder metaDataBuilder = MetaData.builder(clusterState.metaData()).put(idxMetaBuilder);
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaDataBuilder).build();
    clusterStates.add(clusterState);
    if (withPrimaryAllocationFailures) {
        boolean alreadyFailedPrimary = false;
        // some primaries failed to allocate
        indexRoutingTable = routingTable.index(indexName);
        newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
        for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
            final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
            for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
                if (shardRouting.primary() && (shardRouting.started() == false || alreadyFailedPrimary == false)) {
                    newIndexRoutingTable.addShard(shardRouting.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "unlucky shard")));
                    alreadyFailedPrimary = true;
                } else {
                    newIndexRoutingTable.addShard(shardRouting);
                }
            }
        }
        routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
        clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
        return clusterStates;
    }
    // all primaries started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() && shardRouting.started() == false) {
                final ShardRouting newShardRouting = shardRouting.moveToStarted();
                allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
                newIndexRoutingTable.addShard(newShardRouting);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    idxMetaBuilder = IndexMetaData.builder(clusterState.metaData().index(indexName));
    for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
        idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
    }
    metaDataBuilder = MetaData.builder(clusterState.metaData()).put(idxMetaBuilder);
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaDataBuilder).build();
    clusterStates.add(clusterState);
    // initialize replicas
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
        Set<String> allocatedNodes = new HashSet<>();
        allocatedNodes.add(primaryNodeId);
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false) {
                // give the replica a different node id than the primary
                String replicaNodeId = randomFrom(Sets.difference(nodeIds, allocatedNodes));
                newIndexRoutingTable.addShard(shardRouting.initialize(replicaNodeId, null, shardRouting.getExpectedShardSize()));
                allocatedNodes.add(replicaNodeId);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    // some replicas started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false && randomBoolean()) {
                newIndexRoutingTable.addShard(shardRouting.moveToStarted());
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    // all replicas started
    boolean replicaStateChanged = false;
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false && shardRouting.started() == false) {
                newIndexRoutingTable.addShard(shardRouting.moveToStarted());
                replicaStateChanged = true;
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    // all of the replicas may have moved to started in the previous phase already
    if (replicaStateChanged) {
        routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
        clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    }
    return clusterStates;
}