Examples with UnassignedInfo org.elasticsearch.cluster.routing.UnassignedInfo used on opensource projects

Example 26 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project crate by crate.

the class ClusterStateCreationUtils method state.

/**
 * Creates cluster state with and index that has one shard and #(replicaStates) replicas
 *
 * @param index              name of the index
 * @param activePrimaryLocal if active primary should coincide with the local node in the cluster state
 * @param primaryState       state of primary
 * @param replicaStates      states of the replicas. length of this array determines also the number of replicas
 */
public static ClusterState state(String index, boolean activePrimaryLocal, ShardRoutingState primaryState, ShardRoutingState... replicaStates) {
    final int numberOfReplicas = replicaStates.length;
    int numberOfNodes = numberOfReplicas + 1;
    if (primaryState == ShardRoutingState.RELOCATING) {
        numberOfNodes++;
    }
    for (ShardRoutingState state : replicaStates) {
        if (state == ShardRoutingState.RELOCATING) {
            numberOfNodes++;
        }
    }
    // we need a non-local master to test shard failures
    numberOfNodes = Math.max(2, numberOfNodes);
    final ShardId shardId = new ShardId(index, "_na_", 0);
    DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
    Set<String> unassignedNodes = new HashSet<>();
    for (int i = 0; i < numberOfNodes + 1; i++) {
        final DiscoveryNode node = newNode(i);
        discoBuilder = discoBuilder.add(node);
        unassignedNodes.add(node.getId());
    }
    discoBuilder.localNodeId(newNode(0).getId());
    // we need a non-local master to test shard failures
    discoBuilder.masterNodeId(newNode(1).getId());
    final int primaryTerm = 1 + randomInt(200);
    IndexMetadata indexMetadata = IndexMetadata.builder(index).settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).put(SETTING_AUTO_EXPAND_REPLICAS, false).put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
    RoutingTable.Builder routing = new RoutingTable.Builder();
    routing.addAsNew(indexMetadata);
    IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
    String primaryNode = null;
    String relocatingNode = null;
    UnassignedInfo unassignedInfo = null;
    if (primaryState != ShardRoutingState.UNASSIGNED) {
        if (activePrimaryLocal) {
            primaryNode = newNode(0).getId();
            unassignedNodes.remove(primaryNode);
        } else {
            Set<String> unassignedNodesExecludingPrimary = new HashSet<>(unassignedNodes);
            unassignedNodesExecludingPrimary.remove(newNode(0).getId());
            primaryNode = selectAndRemove(unassignedNodesExecludingPrimary);
            unassignedNodes.remove(primaryNode);
        }
        if (primaryState == ShardRoutingState.RELOCATING) {
            relocatingNode = selectAndRemove(unassignedNodes);
        }
    } else {
        unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
    }
    indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, true, primaryState, unassignedInfo));
    for (ShardRoutingState replicaState : replicaStates) {
        String replicaNode = null;
        relocatingNode = null;
        unassignedInfo = null;
        if (replicaState != ShardRoutingState.UNASSIGNED) {
            assert primaryNode != null : "a replica is assigned but the primary isn't";
            replicaNode = selectAndRemove(unassignedNodes);
            if (replicaState == ShardRoutingState.RELOCATING) {
                relocatingNode = selectAndRemove(unassignedNodes);
            }
        } else {
            unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
        }
        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState, unassignedInfo));
    }
    final IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build();
    IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexMetadata);
    indexMetadataBuilder.putInSyncAllocationIds(0, indexShardRoutingTable.activeShards().stream().map(ShardRouting::allocationId).map(AllocationId::getId).collect(Collectors.toSet()));
    ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
    state.nodes(discoBuilder);
    state.metadata(Metadata.builder().put(indexMetadataBuilder.build(), false).generateClusterUuidIfNeeded());
    state.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(indexMetadata.getIndex()).addIndexShard(indexShardRoutingTable)).build());
    return state.build();
}

Example 27 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project crate by crate.

the class AllocationService method resetFailedAllocationCounter.

/**
 * Reset failed allocation counter for unassigned shards
 */
private void resetFailedAllocationCounter(RoutingAllocation allocation) {
    final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
    while (unassignedIterator.hasNext()) {
        ShardRouting shardRouting = unassignedIterator.next();
        UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
        unassignedIterator.updateUnassigned(new UnassignedInfo(unassignedInfo.getNumFailedAllocations() > 0 ? UnassignedInfo.Reason.MANUAL_ALLOCATION : unassignedInfo.getReason(), unassignedInfo.getMessage(), unassignedInfo.getFailure(), 0, unassignedInfo.getUnassignedTimeInNanos(), unassignedInfo.getUnassignedTimeInMillis(), unassignedInfo.isDelayed(), unassignedInfo.getLastAllocationStatus(), Collections.emptySet()), shardRouting.recoverySource(), allocation.changes());
    }
}

Example 28 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project crate by crate.

the class AllocationService method removeDelayMarkers.

/**
 * Removes delay markers from unassigned shards based on current time stamp.
 */
private void removeDelayMarkers(RoutingAllocation allocation) {
    final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
    final Metadata metadata = allocation.metadata();
    while (unassignedIterator.hasNext()) {
        ShardRouting shardRouting = unassignedIterator.next();
        UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
        if (unassignedInfo.isDelayed()) {
            final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(allocation.getCurrentNanoTime(), metadata.getIndexSafe(shardRouting.index()).getSettings());
            if (newComputedLeftDelayNanos == 0) {
                unassignedIterator.updateUnassigned(new UnassignedInfo(unassignedInfo.getReason(), unassignedInfo.getMessage(), unassignedInfo.getFailure(), unassignedInfo.getNumFailedAllocations(), unassignedInfo.getUnassignedTimeInNanos(), unassignedInfo.getUnassignedTimeInMillis(), false, unassignedInfo.getLastAllocationStatus(), unassignedInfo.getFailedNodeIds()), shardRouting.recoverySource(), allocation.changes());
            }
        }
    }
}

Example 29 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project crate by crate.

the class DiskThresholdDeciderUnitTests method testSizeShrinkIndex.

@Test
public void testSizeShrinkIndex() {
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    shardSizes.put("[test][0][p]", 10L);
    shardSizes.put("[test][1][p]", 100L);
    shardSizes.put("[test][2][p]", 500L);
    shardSizes.put("[test][3][p]", 500L);
    ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
    Metadata.Builder metaBuilder = Metadata.builder();
    metaBuilder.put(IndexMetadata.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234")).numberOfShards(4).numberOfReplicas(0));
    metaBuilder.put(IndexMetadata.builder("target").settings(settings(Version.CURRENT).put("index.uuid", "5678").put(IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY, "test").put(IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY, "1234")).numberOfShards(1).numberOfReplicas(0));
    metaBuilder.put(IndexMetadata.builder("target2").settings(settings(Version.CURRENT).put("index.uuid", "9101112").put(IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY, "test").put(IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY, "1234")).numberOfShards(2).numberOfReplicas(0));
    Metadata metadata = metaBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    routingTableBuilder.addAsNew(metadata.index("test"));
    routingTableBuilder.addAsNew(metadata.index("target"));
    routingTableBuilder.addAsNew(metadata.index("target2"));
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTableBuilder.build()).build();
    AllocationService allocationService = createAllocationService();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = allocationService.reroute(clusterState, "foo");
    clusterState = startShardsAndReroute(allocationService, clusterState, clusterState.getRoutingTable().index("test").shardsWithState(ShardRoutingState.UNASSIGNED));
    RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0);
    final Index index = new Index("test", "1234");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, "node1");
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, "node2");
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_2 = ShardRoutingHelper.initialize(test_2, "node1");
    ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_3 = ShardRoutingHelper.initialize(test_3, "node1");
    assertEquals(500L, getExpectedShardSize(test_3, 0L, allocation));
    assertEquals(500L, getExpectedShardSize(test_2, 0L, allocation));
    assertEquals(100L, getExpectedShardSize(test_1, 0L, allocation));
    assertEquals(10L, getExpectedShardSize(test_0, 0L, allocation));
    ShardRouting target = ShardRouting.newUnassigned(new ShardId(new Index("target", "5678"), 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(1110L, getExpectedShardSize(target, 0L, allocation));
    ShardRouting target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(110L, getExpectedShardSize(target2, 0L, allocation));
    target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 1), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(1000L, getExpectedShardSize(target2, 0L, allocation));
    // check that the DiskThresholdDecider still works even if the source index has been deleted
    ClusterState clusterStateWithMissingSourceIndex = ClusterState.builder(clusterState).metadata(Metadata.builder(metadata).remove("test")).routingTable(RoutingTable.builder(clusterState.routingTable()).remove("test").build()).build();
    allocationService.reroute(clusterState, "foo");
    RoutingAllocation allocationWithMissingSourceIndex = new RoutingAllocation(null, clusterStateWithMissingSourceIndex.getRoutingNodes(), clusterStateWithMissingSourceIndex, info, 0);
    assertEquals(42L, getExpectedShardSize(target, 42L, allocationWithMissingSourceIndex));
    assertEquals(42L, getExpectedShardSize(target2, 42L, allocationWithMissingSourceIndex));
}

Example 30 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project crate by crate.

the class DiskThresholdDeciderUnitTests method testCannotAllocateDueToLackOfDiskResources.

@Test
public void testCannotAllocateDueToLackOfDiskResources() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    final Index index = metadata.index("test").getIndex();
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node_0).add(node_1)).build();
    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    // all full
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0));
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    final int freeBytes = randomIntBetween(20, 100);
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, freeBytes));
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // way bigger than available space
    final long shardSize = randomIntBetween(110, 1000);
    shardSizes.put("[test][0][p]", shardSize);
    ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
    allocation.debugDecision(true);
    Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(Decision.Type.NO, decision.type());
    assertThat(decision.getExplanation(), containsString("allocating the shard to this node will bring the node above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=90%] " + "and cause it to have less than the minimum required [0b] of free space " + "(free: [" + freeBytes + "b], estimated shard size: [" + shardSize + "b])"));
}