Examples with MoveAllocationCommand org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand used on opensource projects

Example 21 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class DeadNodesAllocationTests method testDeadNodeWhileRelocatingOnToNode.

public void testDeadNodeWhileRelocatingOnToNode() {
    AllocationService allocation = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").build());
    logger.info("--> building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("--> adding 2 nodes on same rack and do rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = allocation.reroute(clusterState, "reroute");
    // starting primaries
    clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    // starting replicas
    clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    logger.info("--> verifying all is allocated");
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
    logger.info("--> adding additional node");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    clusterState = allocation.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
    assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
    assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
    String origPrimaryNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
    String origReplicaNodeId = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
    logger.info("--> moving primary shard to node3");
    AllocationService.CommandsResult commandsResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")), false, false);
    assertThat(commandsResult.getClusterState(), not(equalTo(clusterState)));
    clusterState = commandsResult.getClusterState();
    assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
    assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
    logger.info("--> fail primary shard recovering instance on node3 being initialized by killing node3");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode(origPrimaryNodeId)).add(newNode(origReplicaNodeId))).build();
    clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
    assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(STARTED));
    assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
}

Example 22 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class RelocationIT method testSimpleRelocationNoIndexing.

public void testSimpleRelocationNoIndexing() {
    logger.info("--> starting [node1] ...");
    final String node_1 = internalCluster().startNode();
    logger.info("--> creating test index ...");
    prepareCreate("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get();
    logger.info("--> index 10 docs");
    for (int i = 0; i < 10; i++) {
        client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
    }
    logger.info("--> flush so we have an actual index");
    client().admin().indices().prepareFlush().execute().actionGet();
    logger.info("--> index more docs so we have something in the translog");
    for (int i = 10; i < 20; i++) {
        client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
    }
    logger.info("--> verifying count");
    client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits(), equalTo(20L));
    logger.info("--> start another node");
    final String node_2 = internalCluster().startNode();
    ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
    logger.info("--> relocate the shard from node1 to node2");
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node_1, node_2)).execute().actionGet();
    clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
    logger.info("--> verifying count again...");
    client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits(), equalTo(20L));
}

Example 23 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class RelocationIT method testRelocationWhileIndexingRandom.

@TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
public void testRelocationWhileIndexingRandom() throws Exception {
    int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
    int numberOfReplicas = randomBoolean() ? 0 : 1;
    int numberOfNodes = numberOfReplicas == 0 ? 2 : 3;
    logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", numberOfRelocations, numberOfReplicas, numberOfNodes);
    String[] nodes = new String[numberOfNodes];
    logger.info("--> starting [node1] ...");
    nodes[0] = internalCluster().startNode();
    logger.info("--> creating test index ...");
    prepareCreate("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", numberOfReplicas)).get();
    for (int i = 2; i <= numberOfNodes; i++) {
        logger.info("--> starting [node{}] ...", i);
        nodes[i - 1] = internalCluster().startNode();
        if (i != numberOfNodes) {
            ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes(Integer.toString(i)).setWaitForGreenStatus().execute().actionGet();
            assertThat(healthResponse.isTimedOut(), equalTo(false));
        }
    }
    int numDocs = scaledRandomIntBetween(200, 2500);
    try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type1", client(), numDocs)) {
        logger.info("--> waiting for {} docs to be indexed ...", numDocs);
        waitForDocs(numDocs, indexer);
        logger.info("--> {} docs indexed", numDocs);
        logger.info("--> starting relocations...");
        // if we have replicas shift those
        int nodeShiftBased = numberOfReplicas;
        for (int i = 0; i < numberOfRelocations; i++) {
            int fromNode = (i % 2);
            int toNode = fromNode == 0 ? 1 : 0;
            fromNode += nodeShiftBased;
            toNode += nodeShiftBased;
            numDocs = scaledRandomIntBetween(200, 1000);
            logger.debug("--> Allow indexer to index [{}] documents", numDocs);
            indexer.continueIndexing(numDocs);
            logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
            client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, nodes[fromNode], nodes[toNode])).get();
            if (rarely()) {
                logger.debug("--> flushing");
                client().admin().indices().prepareFlush().get();
            }
            ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
            assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
            indexer.pauseIndexing();
            logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode);
        }
        logger.info("--> done relocations");
        logger.info("--> waiting for indexing threads to stop ...");
        indexer.stop();
        logger.info("--> indexing threads stopped");
        logger.info("--> refreshing the index");
        client().admin().indices().prepareRefresh("test").execute().actionGet();
        logger.info("--> searching the index");
        boolean ranOnce = false;
        for (int i = 0; i < 10; i++) {
            logger.info("--> START search test round {}", i + 1);
            SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).storedFields().execute().actionGet().getHits();
            ranOnce = true;
            if (hits.getTotalHits() != indexer.totalIndexedDocs()) {
                int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
                for (int hit = 0; hit < indexer.totalIndexedDocs(); hit++) {
                    hitIds[hit] = hit + 1;
                }
                IntHashSet set = IntHashSet.from(hitIds);
                for (SearchHit hit : hits.getHits()) {
                    int id = Integer.parseInt(hit.getId());
                    if (!set.remove(id)) {
                        logger.error("Extra id [{}]", id);
                    }
                }
                set.forEach((IntProcedure) value -> {
                    logger.error("Missing id [{}]", value);
                });
            }
            assertThat(hits.getTotalHits(), equalTo(indexer.totalIndexedDocs()));
            logger.info("--> DONE search test round {}", i + 1);
        }
        if (!ranOnce) {
            fail();
        }
    }
}

Example 24 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class RelocationIT method testRelocationWhileRefreshing.

@TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
public void testRelocationWhileRefreshing() throws Exception {
    int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
    int numberOfReplicas = randomBoolean() ? 0 : 1;
    int numberOfNodes = numberOfReplicas == 0 ? 2 : 3;
    logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", numberOfRelocations, numberOfReplicas, numberOfNodes);
    String[] nodes = new String[numberOfNodes];
    logger.info("--> starting [node_0] ...");
    nodes[0] = internalCluster().startNode();
    logger.info("--> creating test index ...");
    prepareCreate("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", numberOfReplicas).put("index.refresh_interval", // we want to control refreshes c
    -1)).get();
    for (int i = 1; i < numberOfNodes; i++) {
        logger.info("--> starting [node_{}] ...", i);
        nodes[i] = internalCluster().startNode();
        if (i != numberOfNodes - 1) {
            ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes(Integer.toString(i + 1)).setWaitForGreenStatus().execute().actionGet();
            assertThat(healthResponse.isTimedOut(), equalTo(false));
        }
    }
    final Semaphore postRecoveryShards = new Semaphore(0);
    final IndexEventListener listener = new IndexEventListener() {

        @Override
        public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) {
            if (currentState == IndexShardState.POST_RECOVERY) {
                postRecoveryShards.release();
            }
        }
    };
    for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getInstances(MockIndexEventListener.TestEventListener.class)) {
        eventListener.setNewDelegate(listener);
    }
    logger.info("--> starting relocations...");
    // if we have replicas shift those
    int nodeShiftBased = numberOfReplicas;
    for (int i = 0; i < numberOfRelocations; i++) {
        int fromNode = (i % 2);
        int toNode = fromNode == 0 ? 1 : 0;
        fromNode += nodeShiftBased;
        toNode += nodeShiftBased;
        List<IndexRequestBuilder> builders1 = new ArrayList<>();
        for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
            builders1.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
        }
        List<IndexRequestBuilder> builders2 = new ArrayList<>();
        for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
            builders2.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
        }
        logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
        client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, nodes[fromNode], nodes[toNode])).get();
        logger.debug("--> index [{}] documents", builders1.size());
        indexRandom(false, true, builders1);
        // wait for shard to reach post recovery
        postRecoveryShards.acquire(1);
        logger.debug("--> index [{}] documents", builders2.size());
        indexRandom(true, true, builders2);
        // verify cluster was finished.
        assertFalse(client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).setWaitForEvents(Priority.LANGUID).setTimeout("30s").get().isTimedOut());
        logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode);
        logger.debug("--> verifying all searches return the same number of docs");
        long expectedCount = -1;
        for (Client client : clients()) {
            SearchResponse response = client.prepareSearch("test").setPreference("_local").setSize(0).get();
            assertNoFailures(response);
            if (expectedCount < 0) {
                expectedCount = response.getHits().getTotalHits();
            } else {
                assertEquals(expectedCount, response.getHits().getTotalHits());
            }
        }
    }
}

Example 25 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class ClusterRerouteIT method testRerouteExplain.

public void testRerouteExplain() {
    Settings commonSettings = Settings.builder().build();
    logger.info("--> starting a node");
    String node_1 = internalCluster().startNode(commonSettings);
    assertThat(cluster().size(), equalTo(1));
    ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("1").execute().actionGet();
    assertThat(healthResponse.isTimedOut(), equalTo(false));
    logger.info("--> create an index with 1 shard");
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).execute().actionGet();
    ensureGreen("test");
    logger.info("--> disable allocation");
    Settings newSettings = Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.NONE.name()).build();
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(newSettings).execute().actionGet();
    logger.info("--> starting a second node");
    String node_2 = internalCluster().startNode(commonSettings);
    assertThat(cluster().size(), equalTo(2));
    healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
    assertThat(healthResponse.isTimedOut(), equalTo(false));
    logger.info("--> try to move the shard from node1 to node2");
    MoveAllocationCommand cmd = new MoveAllocationCommand("test", 0, node_1, node_2);
    ClusterRerouteResponse resp = client().admin().cluster().prepareReroute().add(cmd).setExplain(true).execute().actionGet();
    RoutingExplanations e = resp.getExplanations();
    assertThat(e.explanations().size(), equalTo(1));
    RerouteExplanation explanation = e.explanations().get(0);
    assertThat(explanation.command().name(), equalTo(cmd.name()));
    assertThat(((MoveAllocationCommand) explanation.command()).shardId(), equalTo(cmd.shardId()));
    assertThat(((MoveAllocationCommand) explanation.command()).fromNode(), equalTo(cmd.fromNode()));
    assertThat(((MoveAllocationCommand) explanation.command()).toNode(), equalTo(cmd.toNode()));
    assertThat(explanation.decisions().type(), equalTo(Decision.Type.YES));
}