Examples with ShardRouting org.elasticsearch.cluster.routing.ShardRouting used on opensource projects

Example 56 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class AckIT method testClusterRerouteAcknowledgement.

public void testClusterRerouteAcknowledgement() throws InterruptedException {
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, between(cluster().numDataNodes(), DEFAULT_MAX_NUM_SHARDS)).put(SETTING_NUMBER_OF_REPLICAS, 0)));
    ensureGreen();
    MoveAllocationCommand moveAllocationCommand = getAllocationCommand();
    final Index index = client().admin().cluster().prepareState().get().getState().metaData().index("test").getIndex();
    final ShardId commandShard = new ShardId(index, moveAllocationCommand.shardId());
    assertAcked(client().admin().cluster().prepareReroute().add(moveAllocationCommand));
    for (Client client : clients()) {
        ClusterState clusterState = getLocalClusterState(client);
        for (ShardRouting shardRouting : clusterState.getRoutingNodes().node(moveAllocationCommand.fromNode())) {
            //if the shard that we wanted to move is still on the same node, it must be relocating
            if (shardRouting.shardId().equals(commandShard)) {
                assertThat(shardRouting.relocating(), equalTo(true));
            }
        }
        boolean found = false;
        for (ShardRouting shardRouting : clusterState.getRoutingNodes().node(moveAllocationCommand.toNode())) {
            if (shardRouting.shardId().equals(commandShard)) {
                assertThat(shardRouting.state(), anyOf(equalTo(ShardRoutingState.INITIALIZING), equalTo(ShardRoutingState.STARTED)));
                found = true;
                break;
            }
        }
        assertThat(found, equalTo(true));
    }
}

Example 57 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class AckIT method testClusterRerouteAcknowledgementDryRun.

public void testClusterRerouteAcknowledgementDryRun() throws InterruptedException {
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, between(cluster().numDataNodes(), DEFAULT_MAX_NUM_SHARDS)).put(SETTING_NUMBER_OF_REPLICAS, 0)).get();
    ensureGreen();
    MoveAllocationCommand moveAllocationCommand = getAllocationCommand();
    final Index index = client().admin().cluster().prepareState().get().getState().metaData().index("test").getIndex();
    final ShardId commandShard = new ShardId(index, moveAllocationCommand.shardId());
    assertAcked(client().admin().cluster().prepareReroute().setDryRun(true).add(moveAllocationCommand));
    //testing only on master with the latest cluster state as we didn't make any change thus we cannot guarantee that
    //all nodes hold the same cluster state version. We only know there was no need to change anything, thus no need for ack on this update.
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    boolean found = false;
    for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().node(moveAllocationCommand.fromNode())) {
        //the shard that we wanted to move is still on the same node, as we had dryRun flag
        if (shardRouting.shardId().equals(commandShard)) {
            assertThat(shardRouting.started(), equalTo(true));
            found = true;
            break;
        }
    }
    assertThat(found, equalTo(true));
    for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().node(moveAllocationCommand.toNode())) {
        if (shardRouting.shardId().equals(commandShard)) {
            fail("shard [" + shardRouting + "] shouldn't be on node [" + moveAllocationCommand.toString() + "]");
        }
    }
}

Example 58 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class ShardSegments method readFrom.

@Override
public void readFrom(StreamInput in) throws IOException {
    shardRouting = new ShardRouting(in);
    int size = in.readVInt();
    if (size == 0) {
        segments = Collections.emptyList();
    } else {
        segments = new ArrayList<>(size);
        for (int i = 0; i < size; i++) {
            segments.add(Segment.readSegment(in));
        }
    }
}

Example 59 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class MetaDataCreateIndexService method validateShrinkIndex.

/**
     * Validates the settings and mappings for shrinking an index.
     * @return the list of nodes at least one instance of the source index shards are allocated
     */
static List<String> validateShrinkIndex(ClusterState state, String sourceIndex, Set<String> targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) {
    if (state.metaData().hasIndex(targetIndexName)) {
        throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex());
    }
    final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex);
    if (sourceMetaData == null) {
        throw new IndexNotFoundException(sourceIndex);
    }
    // ensure index is read-only
    if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) {
        throw new IllegalStateException("index " + sourceIndex + " must be read-only to shrink index. use \"index.blocks.write=true\"");
    }
    if (sourceMetaData.getNumberOfShards() == 1) {
        throw new IllegalArgumentException("can't shrink an index with only one shard");
    }
    if ((targetIndexMappingsTypes.size() > 1 || (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) {
        throw new IllegalArgumentException("mappings are not allowed when shrinking indices" + ", all mappings are copied from the source index");
    }
    if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
        // this method applies all necessary checks ie. if the target shards are less than the source shards
        // of if the source shards are divisible by the number of target shards
        IndexMetaData.getRoutingFactor(sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
    }
    // now check that index is all on one node
    final IndexRoutingTable table = state.routingTable().index(sourceIndex);
    Map<String, AtomicInteger> nodesToNumRouting = new HashMap<>();
    int numShards = sourceMetaData.getNumberOfShards();
    for (ShardRouting routing : table.shardsWithState(ShardRoutingState.STARTED)) {
        nodesToNumRouting.computeIfAbsent(routing.currentNodeId(), (s) -> new AtomicInteger(0)).incrementAndGet();
    }
    List<String> nodesToAllocateOn = new ArrayList<>();
    for (Map.Entry<String, AtomicInteger> entries : nodesToNumRouting.entrySet()) {
        int numAllocations = entries.getValue().get();
        assert numAllocations <= numShards : "wait what? " + numAllocations + " is > than num shards " + numShards;
        if (numAllocations == numShards) {
            nodesToAllocateOn.add(entries.getKey());
        }
    }
    if (nodesToAllocateOn.isEmpty()) {
        throw new IllegalStateException("index " + sourceIndex + " must have all shards allocated on the same node to shrink index");
    }
    return nodesToAllocateOn;
}

Example 60 with ShardRouting

use of org.elasticsearch.cluster.routing.ShardRouting in project elasticsearch by elastic.

the class IndexMetaDataUpdater method updateInSyncAllocations.

/**
     * Updates in-sync allocations with routing changes that were made to the routing table.
     */
private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTable, IndexMetaData oldIndexMetaData, IndexMetaData.Builder indexMetaDataBuilder, ShardId shardId, Updates updates) {
    assert Sets.haveEmptyIntersection(updates.addedAllocationIds, updates.removedAllocationIds) : "allocation ids cannot be both added and removed in the same allocation round, added ids: " + updates.addedAllocationIds + ", removed ids: " + updates.removedAllocationIds;
    Set<String> oldInSyncAllocationIds = oldIndexMetaData.inSyncAllocationIds(shardId.id());
    // check if we have been force-initializing an empty primary or a stale primary
    if (updates.initializedPrimary != null && oldInSyncAllocationIds.isEmpty() == false && oldInSyncAllocationIds.contains(updates.initializedPrimary.allocationId().getId()) == false) {
        // we're not reusing an existing in-sync allocation id to initialize a primary, which means that we're either force-allocating
        // an empty or a stale primary (see AllocateEmptyPrimaryAllocationCommand or AllocateStalePrimaryAllocationCommand).
        RecoverySource.Type recoverySourceType = updates.initializedPrimary.recoverySource().getType();
        boolean emptyPrimary = recoverySourceType == RecoverySource.Type.EMPTY_STORE;
        assert updates.addedAllocationIds.isEmpty() : (emptyPrimary ? "empty" : "stale") + " primary is not force-initialized in same allocation round where shards are started";
        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        if (emptyPrimary) {
            // forcing an empty primary resets the in-sync allocations to the empty set (ShardRouting.allocatedPostIndexCreate)
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.emptySet());
        } else {
            // forcing a stale primary resets the in-sync allocations to the singleton set with the stale id
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.singleton(updates.initializedPrimary.allocationId().getId()));
        }
    } else {
        // standard path for updating in-sync ids
        Set<String> inSyncAllocationIds = new HashSet<>(oldInSyncAllocationIds);
        inSyncAllocationIds.addAll(updates.addedAllocationIds);
        inSyncAllocationIds.removeAll(updates.removedAllocationIds);
        // Prevent set of inSyncAllocationIds to grow unboundedly. This can happen for example if we don't write to a primary
        // but repeatedly shut down nodes that have active replicas.
        // We use number_of_replicas + 1 (= possible active shard copies) to bound the inSyncAllocationIds set
        // Only trim the set of allocation ids when it grows, otherwise we might trim too eagerly when the number
        // of replicas was decreased while shards were unassigned.
        // +1 for the primary
        int maxActiveShards = oldIndexMetaData.getNumberOfReplicas() + 1;
        IndexShardRoutingTable newShardRoutingTable = newRoutingTable.shardRoutingTable(shardId);
        if (inSyncAllocationIds.size() > oldInSyncAllocationIds.size() && inSyncAllocationIds.size() > maxActiveShards) {
            // trim entries that have no corresponding shard routing in the cluster state (i.e. trim unavailable copies)
            List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards();
            assert assignedShards.size() <= maxActiveShards : "cannot have more assigned shards " + assignedShards + " than maximum possible active shards " + maxActiveShards;
            Set<String> assignedAllocations = assignedShards.stream().map(s -> s.allocationId().getId()).collect(Collectors.toSet());
            inSyncAllocationIds = inSyncAllocationIds.stream().sorted(// values with routing entries first
            Comparator.comparing(assignedAllocations::contains).reversed()).limit(maxActiveShards).collect(Collectors.toSet());
        }
        // in-sync set, this could create an empty primary on the next allocation.
        if (newShardRoutingTable.activeShards().isEmpty() && updates.firstFailedPrimary != null) {
            // add back allocation id of failed primary
            inSyncAllocationIds.add(updates.firstFailedPrimary.allocationId().getId());
        }
        assert inSyncAllocationIds.isEmpty() == false || oldInSyncAllocationIds.isEmpty() : "in-sync allocations cannot become empty after they have been non-empty: " + oldInSyncAllocationIds;
        // be extra safe here and only update in-sync set if it is non-empty
        if (inSyncAllocationIds.isEmpty() == false) {
            if (indexMetaDataBuilder == null) {
                indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
            }
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), inSyncAllocationIds);
        }
    }
    return indexMetaDataBuilder;
}