Examples with RoutingNode org.elasticsearch.cluster.routing.RoutingNode used on opensource projects

Example 26 with RoutingNode

use of org.elasticsearch.cluster.routing.RoutingNode in project elasticsearch by elastic.

the class DiskThresholdDeciderTests method testFreeDiskPercentageAfterShardAssigned.

public void testFreeDiskPercentageAfterShardAssigned() {
    RoutingNode rn = new RoutingNode("node1", newNode("node1"));
    DiskThresholdDecider decider = makeDecider(Settings.EMPTY);
    Map<String, DiskUsage> usages = new HashMap<>();
    // 50% used
    usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 50));
    // 100% used
    usages.put("node3", new DiskUsage("node3", "n3", "/dev/null", 100, 0));
    Double after = decider.freeDiskPercentageAfterShardAssigned(new DiskUsage("node2", "n2", "/dev/null", 100, 30), 11L);
    assertThat(after, equalTo(19.0));
}

Example 27 with RoutingNode

use of org.elasticsearch.cluster.routing.RoutingNode in project elasticsearch by elastic.

the class DiskThresholdDeciderUnitTests method testCanRemainUsesLeastAvailableSpace.

public void testCanRemainUsesLeastAvailableSpace() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);
    ImmutableOpenMap.Builder<ShardRouting, String> shardRoutingMap = ImmutableOpenMap.builder();
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    final IndexMetaData indexMetaData = metaData.index("test");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId());
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    shardRoutingMap.put(test_0, "/node0/least");
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId());
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    shardRoutingMap.put(test_1, "/node1/least");
    ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId());
    test_2 = ShardRoutingHelper.moveToStarted(test_2);
    shardRoutingMap.put(test_2, "/node1/most");
    ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId());
    test_3 = ShardRoutingHelper.moveToStarted(test_3);
    // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is.
    RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("--> adding two nodes");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node_0).add(node_1)).build();
    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    // 90% used
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "/node0/least", 100, 10));
    // 91% used
    leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "/node1/least", 100, 9));
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    // 10% used
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "/node0/most", 100, 90));
    // 10% used
    mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "/node1/most", 100, 90));
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // 10 bytes
    shardSizes.put("[test][0][p]", 10L);
    shardSizes.put("[test][1][p]", 10L);
    shardSizes.put("[test][2][p]", 10L);
    final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
    allocation.debugDecision(true);
    Decision decision = decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("there is enough disk on this node for the shard to remain, free: [10b]"));
    decision = decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation);
    assertEquals(Decision.Type.NO, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("the shard cannot remain on this node because it is " + "above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%] and there is less than " + "the required [10.0%] free disk on node, actual free: [9.0%]"));
    try {
        decider.canRemain(test_0, new RoutingNode("node_1", node_1), allocation);
        fail("not allocated on this node");
    } catch (IllegalArgumentException ex) {
    // not allocated on that node
    }
    try {
        decider.canRemain(test_1, new RoutingNode("node_0", node_0), allocation);
        fail("not allocated on this node");
    } catch (IllegalArgumentException ex) {
    // not allocated on that node
    }
    decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
    assertEquals("can stay since allocated on a different path with enough space", Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain"));
    decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
    assertEquals("can stay since we don't have information about this shard", Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain"));
}

Example 28 with RoutingNode

use of org.elasticsearch.cluster.routing.RoutingNode in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testMultiIndexEvenDistribution.

public void testMultiIndexEvenDistribution() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    final int numberOfIndices = 50;
    logger.info("Building initial routing table with " + numberOfIndices + " indices");
    MetaData.Builder metaDataBuilder = MetaData.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        metaDataBuilder.put(IndexMetaData.builder("test" + i).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
    }
    MetaData metaData = metaDataBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        routingTableBuilder.addAsNew(metaData.index("test" + i));
    }
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    assertThat(clusterState.routingTable().indicesRouting().size(), equalTo(numberOfIndices));
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Adding " + (numberOfIndices / 2) + " nodes");
    DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
    List<DiscoveryNode> nodes = new ArrayList<>();
    for (int i = 0; i < (numberOfIndices / 2); i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards initializing per node on the first 25 nodes
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    Set<String> encounteredIndices = new HashSet<>();
    for (RoutingNode routingNode : routingNodes) {
        assertThat(routingNode.numberOfShardsWithState(STARTED), equalTo(0));
        assertThat(routingNode.size(), equalTo(2));
        // make sure we still have 2 shards initializing per node on the only 25 nodes
        int nodeIndex = Integer.parseInt(routingNode.nodeId().substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
        // check that we don't have a shard associated with a node with the same index name (we have a single shard)
        for (ShardRouting shardRoutingEntry : routingNode) {
            assertThat(encounteredIndices, not(hasItem(shardRoutingEntry.getIndexName())));
            encounteredIndices.add(shardRoutingEntry.getIndexName());
        }
    }
    logger.info("Adding additional " + (numberOfIndices / 2) + " nodes, nothing should change");
    nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
    for (int i = (numberOfIndices / 2); i < numberOfIndices; i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Marking the shard as started");
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    int numberOfRelocatingShards = 0;
    int numberOfStartedShards = 0;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), anyOf(equalTo(STARTED), equalTo(RELOCATING)));
        if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == STARTED) {
            numberOfStartedShards++;
        } else if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == RELOCATING) {
            numberOfRelocatingShards++;
        }
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards either relocating or started on the first 25 nodes (still)
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    assertThat(numberOfRelocatingShards, equalTo(25));
    assertThat(numberOfStartedShards, equalTo(25));
}

Example 29 with RoutingNode

use of org.elasticsearch.cluster.routing.RoutingNode in project crate by crate.

the class BlobRecoverySource method recover.

private RecoveryResponse recover(final StartRecoveryRequest request) {
    final IndexService indexService = indicesService.indexServiceSafe(request.shardId().index().name());
    final IndexShard shard = indexService.shardSafe(request.shardId().id());
    // starting recovery from that our (the source) shard state is marking the shard to be in recovery mode as well, otherwise
    // the index operations will not be routed to it properly
    RoutingNode node = clusterService.state().getRoutingNodes().node(request.targetNode().getId());
    if (node == null) {
        logger.debug("delaying recovery of {} as source node {} is unknown", request.shardId(), request.targetNode());
        throw new DelayRecoveryException("source node does not have the node [" + request.targetNode() + "] in its state yet..");
    }
    ShardRouting targetShardRouting = null;
    for (ShardRouting shardRouting : node) {
        if (shardRouting.shardId().equals(request.shardId())) {
            targetShardRouting = shardRouting;
            break;
        }
    }
    if (targetShardRouting == null) {
        logger.debug("delaying recovery of {} as it is not listed as assigned to target node {}", request.shardId(), request.targetNode());
        throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node");
    }
    if (!targetShardRouting.initializing()) {
        logger.debug("delaying recovery of {} as it is not listed as initializing on the target node {}. known shards state is [{}]", request.shardId(), request.targetNode(), targetShardRouting.state());
        throw new DelayRecoveryException("source node has the state of the target shard to be [" + targetShardRouting.state() + "], expecting to be [initializing]");
    }
    logger.trace("[{}][{}] starting recovery to {}, mark_as_relocated {}", request.shardId().index().name(), request.shardId().id(), request.targetNode(), request.markAsRelocated());
    final RecoverySourceHandler handler;
    if (IndexMetaData.isOnSharedFilesystem(shard.indexSettings())) {
        handler = new SharedFSRecoverySourceHandler(shard, request, recoverySettings, transportService, logger);
    } else {
        // CRATE CHANGE:
        handler = new BlobRecoverySourceHandler(shard, request, recoverySettings, transportService, logger, blobTransferTarget, blobIndicesService);
    }
    ongoingRecoveries.add(shard, handler);
    try {
        return handler.recoverToTarget();
    } finally {
        ongoingRecoveries.remove(shard, handler);
    }
}

Example 30 with RoutingNode

use of org.elasticsearch.cluster.routing.RoutingNode in project elasticsearch by elastic.

the class IndicesClusterStateService method updateIndices.

private void updateIndices(ClusterChangedEvent event) {
    if (!event.metaDataChanged()) {
        return;
    }
    final ClusterState state = event.state();
    for (AllocatedIndex<? extends Shard> indexService : indicesService) {
        final Index index = indexService.index();
        final IndexMetaData currentIndexMetaData = indexService.getIndexSettings().getIndexMetaData();
        final IndexMetaData newIndexMetaData = state.metaData().index(index);
        assert newIndexMetaData != null : "index " + index + " should have been removed by deleteIndices";
        if (ClusterChangedEvent.indexMetaDataChanged(currentIndexMetaData, newIndexMetaData)) {
            indexService.updateMetaData(newIndexMetaData);
            try {
                if (indexService.updateMapping(newIndexMetaData) && sendRefreshMapping) {
                    nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(newIndexMetaData.getIndex().getName(), newIndexMetaData.getIndexUUID(), state.nodes().getLocalNodeId()));
                }
            } catch (Exception e) {
                indicesService.removeIndex(indexService.index(), FAILURE, "removing index (mapping update failed)");
                // fail shards that would be created or updated by createOrUpdateShards
                RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
                if (localRoutingNode != null) {
                    for (final ShardRouting shardRouting : localRoutingNode) {
                        if (shardRouting.index().equals(index) && failedShardsCache.containsKey(shardRouting.shardId()) == false) {
                            sendFailShard(shardRouting, "failed to update mapping for index", e, state);
                        }
                    }
                }
            }
        }
    }
}