Examples with IndexShardRoutingTable org.elasticsearch.cluster.routing.IndexShardRoutingTable used on opensource projects

Example 36 with IndexShardRoutingTable

use of org.elasticsearch.cluster.routing.IndexShardRoutingTable in project elasticsearch by elastic.

the class IndicesStore method clusterChanged.

@Override
public void clusterChanged(ClusterChangedEvent event) {
    if (!event.routingTableChanged()) {
        return;
    }
    if (event.state().blocks().disableStatePersistence()) {
        return;
    }
    RoutingTable routingTable = event.state().routingTable();
    // - closed indices don't need to be removed from the cache but we do it anyway for code simplicity
    for (Iterator<ShardId> it = folderNotFoundCache.iterator(); it.hasNext(); ) {
        ShardId shardId = it.next();
        if (routingTable.hasIndex(shardId.getIndex()) == false) {
            it.remove();
        }
    }
    // remove entries from cache which are allocated to this node
    final String localNodeId = event.state().nodes().getLocalNodeId();
    RoutingNode localRoutingNode = event.state().getRoutingNodes().node(localNodeId);
    if (localRoutingNode != null) {
        for (ShardRouting routing : localRoutingNode) {
            folderNotFoundCache.remove(routing.shardId());
        }
    }
    for (IndexRoutingTable indexRoutingTable : routingTable) {
        // Note, closed indices will not have any routing information, so won't be deleted
        for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
            ShardId shardId = indexShardRoutingTable.shardId();
            if (folderNotFoundCache.contains(shardId) == false && shardCanBeDeleted(localNodeId, indexShardRoutingTable)) {
                IndexService indexService = indicesService.indexService(indexRoutingTable.getIndex());
                final IndexSettings indexSettings;
                if (indexService == null) {
                    IndexMetaData indexMetaData = event.state().getMetaData().getIndexSafe(indexRoutingTable.getIndex());
                    indexSettings = new IndexSettings(indexMetaData, settings);
                } else {
                    indexSettings = indexService.getIndexSettings();
                }
                IndicesService.ShardDeletionCheckResult shardDeletionCheckResult = indicesService.canDeleteShardContent(shardId, indexSettings);
                switch(shardDeletionCheckResult) {
                    case FOLDER_FOUND_CAN_DELETE:
                        deleteShardIfExistElseWhere(event.state(), indexShardRoutingTable);
                        break;
                    case NO_FOLDER_FOUND:
                        folderNotFoundCache.add(shardId);
                        break;
                    case NO_LOCAL_STORAGE:
                        assert false : "shard deletion only runs on data nodes which always have local storage";
                        // nothing to do
                        break;
                    case STILL_ALLOCATED:
                        // nothing to do
                        break;
                    case SHARED_FILE_SYSTEM:
                        // nothing to do
                        break;
                    default:
                        assert false : "unknown shard deletion check result: " + shardDeletionCheckResult;
                }
            }
        }
    }
}

Example 37 with IndexShardRoutingTable

use of org.elasticsearch.cluster.routing.IndexShardRoutingTable in project elasticsearch by elastic.

the class ReplicationOperation method getShards.

protected List<ShardRouting> getShards(ShardId shardId, ClusterState state) {
    // can be null if the index is deleted / closed on us..
    final IndexShardRoutingTable shardRoutingTable = state.getRoutingTable().shardRoutingTableOrNull(shardId);
    List<ShardRouting> shards = shardRoutingTable == null ? Collections.emptyList() : shardRoutingTable.shards();
    return shards;
}

Example 38 with IndexShardRoutingTable

use of org.elasticsearch.cluster.routing.IndexShardRoutingTable in project elasticsearch by elastic.

the class CatAllocationTestCase method testRun.

public void testRun() throws IOException {
    Set<String> nodes = new HashSet<>();
    Map<String, Idx> indices = new HashMap<>();
    try (BufferedReader reader = Files.newBufferedReader(getCatPath(), StandardCharsets.UTF_8)) {
        String line = null;
        // regexp FTW
        Pattern pattern = Pattern.compile("^(.+)\\s+(\\d)\\s+([rp])\\s+(STARTED|RELOCATING|INITIALIZING|UNASSIGNED)" + "\\s+\\d+\\s+[0-9.a-z]+\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+).*$");
        while ((line = reader.readLine()) != null) {
            final Matcher matcher;
            if ((matcher = pattern.matcher(line)).matches()) {
                final String index = matcher.group(1);
                Idx idx = indices.get(index);
                if (idx == null) {
                    idx = new Idx(index);
                    indices.put(index, idx);
                }
                final int shard = Integer.parseInt(matcher.group(2));
                final boolean primary = matcher.group(3).equals("p");
                ShardRoutingState state = ShardRoutingState.valueOf(matcher.group(4));
                String ip = matcher.group(5);
                nodes.add(ip);
                ShardRouting routing = TestShardRouting.newShardRouting(index, shard, ip, null, primary, state);
                idx.add(routing);
                logger.debug("Add routing {}", routing);
            } else {
                fail("can't read line: " + line);
            }
        }
    }
    logger.info("Building initial routing table");
    MetaData.Builder builder = MetaData.builder();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (Idx idx : indices.values()) {
        IndexMetaData.Builder idxMetaBuilder = IndexMetaData.builder(idx.name).settings(settings(Version.CURRENT)).numberOfShards(idx.numShards()).numberOfReplicas(idx.numReplicas());
        for (ShardRouting shardRouting : idx.routing) {
            if (shardRouting.active()) {
                Set<String> allocationIds = idxMetaBuilder.getInSyncAllocationIds(shardRouting.id());
                if (allocationIds == null) {
                    allocationIds = new HashSet<>();
                } else {
                    allocationIds = new HashSet<>(allocationIds);
                }
                allocationIds.add(shardRouting.allocationId().getId());
                idxMetaBuilder.putInSyncAllocationIds(shardRouting.id(), allocationIds);
            }
        }
        IndexMetaData idxMeta = idxMetaBuilder.build();
        builder.put(idxMeta, false);
        IndexRoutingTable.Builder tableBuilder = new IndexRoutingTable.Builder(idxMeta.getIndex()).initializeAsRecovery(idxMeta);
        Map<Integer, IndexShardRoutingTable> shardIdToRouting = new HashMap<>();
        for (ShardRouting r : idx.routing) {
            IndexShardRoutingTable refData = new IndexShardRoutingTable.Builder(r.shardId()).addShard(r).build();
            if (shardIdToRouting.containsKey(r.getId())) {
                refData = new IndexShardRoutingTable.Builder(shardIdToRouting.get(r.getId())).addShard(r).build();
            }
            shardIdToRouting.put(r.getId(), refData);
        }
        for (IndexShardRoutingTable t : shardIdToRouting.values()) {
            tableBuilder.addIndexShard(t);
        }
        IndexRoutingTable table = tableBuilder.build();
        routingTableBuilder.add(table);
    }
    MetaData metaData = builder.build();
    RoutingTable routingTable = routingTableBuilder.build();
    DiscoveryNodes.Builder builderDiscoNodes = DiscoveryNodes.builder();
    for (String node : nodes) {
        builderDiscoNodes.add(newNode(node));
    }
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).nodes(builderDiscoNodes.build()).build();
    if (balanceFirst()) {
        clusterState = rebalance(clusterState);
    }
    clusterState = allocateNew(clusterState);
}

Example 39 with IndexShardRoutingTable

use of org.elasticsearch.cluster.routing.IndexShardRoutingTable in project elasticsearch by elastic.

the class InSyncAllocationIdTests method testInSyncIdsNotGrowingWithoutBounds.

/**
     * Prevent set of inSyncAllocationIds to grow unboundedly. This can happen for example if we don't write to a primary
     * but repeatedly shut down nodes that have active replicas.
     * We use number_of_replicas + 1 (= possible active shard copies) to bound the inSyncAllocationIds set
     */
public void testInSyncIdsNotGrowingWithoutBounds() throws Exception {
    ClusterState clusterState = createOnePrimaryOneReplicaClusterState(allocation);
    Set<String> inSyncSet = clusterState.metaData().index("test").inSyncAllocationIds(0);
    assertThat(inSyncSet.size(), equalTo(2));
    IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index("test").shard(0);
    ShardRouting primaryShard = shardRoutingTable.primaryShard();
    ShardRouting replicaShard = shardRoutingTable.replicaShards().get(0);
    logger.info("remove a node");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(replicaShard.currentNodeId())).build();
    clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
    // in-sync allocation ids should not be updated
    assertEquals(inSyncSet, clusterState.metaData().index("test").inSyncAllocationIds(0));
    // check that inSyncAllocationIds can not grow without bounds
    for (int i = 0; i < 5; i++) {
        logger.info("add back node");
        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(replicaShard.currentNodeId()))).build();
        clusterState = allocation.reroute(clusterState, "reroute");
        logger.info("start replica shards");
        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
        logger.info("remove the node");
        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(replicaShard.currentNodeId())).build();
        clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
    }
    // in-sync allocation set is bounded
    Set<String> newInSyncSet = clusterState.metaData().index("test").inSyncAllocationIds(0);
    assertThat(newInSyncSet.size(), equalTo(2));
    // only allocation id of replica was changed
    assertFalse(Sets.haveEmptyIntersection(inSyncSet, newInSyncSet));
    assertThat(newInSyncSet, hasItem(primaryShard.allocationId().getId()));
}

Example 40 with IndexShardRoutingTable

use of org.elasticsearch.cluster.routing.IndexShardRoutingTable in project elasticsearch by elastic.

the class InSyncAllocationIdTests method testDeadNodesBeforeReplicaFailed.

/**
     * Assume following scenario: indexing request is written to primary, but fails to be replicated to active replica.
     * The primary instructs master to fail replica before acknowledging write to client. In the meanwhile, the node of the replica was
     * removed from the cluster (deassociateDeadNodes). This means that the ShardRouting of the replica was failed, but it's allocation
     * id is still part of the in-sync set. We have to make sure that the failShard request from the primary removes the allocation id
     * from the in-sync set.
     */
public void testDeadNodesBeforeReplicaFailed() throws Exception {
    ClusterState clusterState = createOnePrimaryOneReplicaClusterState(allocation);
    logger.info("remove replica node");
    IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index("test").shard(0);
    ShardRouting replicaShard = shardRoutingTable.replicaShards().get(0);
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(replicaShard.currentNodeId())).build();
    clusterState = allocation.deassociateDeadNodes(clusterState, true, "reroute");
    assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(2));
    logger.info("fail replica (for which there is no shard routing in the CS anymore)");
    assertNull(clusterState.getRoutingNodes().getByAllocationId(replicaShard.shardId(), replicaShard.allocationId().getId()));
    ShardStateAction.ShardFailedClusterStateTaskExecutor failedClusterStateTaskExecutor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocation, null, logger);
    long primaryTerm = clusterState.metaData().index("test").primaryTerm(0);
    clusterState = failedClusterStateTaskExecutor.execute(clusterState, Arrays.asList(new ShardEntry(shardRoutingTable.shardId(), replicaShard.allocationId().getId(), primaryTerm, "dummy", null))).resultingState;
    assertThat(clusterState.metaData().index("test").inSyncAllocationIds(0).size(), equalTo(1));
}