Examples with ShardIterator org.elasticsearch.cluster.routing.ShardIterator used on opensource projects

Example 6 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class RoutingIteratorTests method testRandomRouting.

public void testRandomRouting() {
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build();
    ShardIterator shardIterator = routingTable.index("test1").shard(0).shardsRandomIt();
    ShardRouting shardRouting1 = shardIterator.nextOrNull();
    assertThat(shardRouting1, notNullValue());
    assertThat(shardIterator.nextOrNull(), notNullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    shardIterator = routingTable.index("test1").shard(0).shardsRandomIt();
    ShardRouting shardRouting2 = shardIterator.nextOrNull();
    assertThat(shardRouting2, notNullValue());
    ShardRouting shardRouting3 = shardIterator.nextOrNull();
    assertThat(shardRouting3, notNullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardRouting1, not(sameInstance(shardRouting2)));
    assertThat(shardRouting1, sameInstance(shardRouting3));
}

Example 7 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class RoutingIteratorTests method testIterator2.

public void testIterator2() {
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build();
    ShardIterator shardIterator = routingTable.index("test1").shard(0).shardsIt(0);
    assertThat(shardIterator.size(), equalTo(2));
    ShardRouting shardRouting1 = shardIterator.nextOrNull();
    assertThat(shardRouting1, notNullValue());
    assertThat(shardIterator.remaining(), equalTo(1));
    ShardRouting shardRouting2 = shardIterator.nextOrNull();
    assertThat(shardRouting2, notNullValue());
    assertThat(shardIterator.remaining(), equalTo(0));
    assertThat(shardRouting2, not(sameInstance(shardRouting1)));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.remaining(), equalTo(0));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.remaining(), equalTo(0));
    shardIterator = routingTable.index("test1").shard(0).shardsIt(1);
    assertThat(shardIterator.size(), equalTo(2));
    ShardRouting shardRouting3 = shardIterator.nextOrNull();
    assertThat(shardRouting1, notNullValue());
    ShardRouting shardRouting4 = shardIterator.nextOrNull();
    assertThat(shardRouting2, notNullValue());
    assertThat(shardRouting2, not(sameInstance(shardRouting1)));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardRouting1, not(sameInstance(shardRouting3)));
    assertThat(shardRouting2, not(sameInstance(shardRouting4)));
    assertThat(shardRouting1, sameInstance(shardRouting4));
    assertThat(shardRouting2, sameInstance(shardRouting3));
    shardIterator = routingTable.index("test1").shard(0).shardsIt(2);
    assertThat(shardIterator.size(), equalTo(2));
    ShardRouting shardRouting5 = shardIterator.nextOrNull();
    assertThat(shardRouting5, notNullValue());
    ShardRouting shardRouting6 = shardIterator.nextOrNull();
    assertThat(shardRouting6, notNullValue());
    assertThat(shardRouting6, not(sameInstance(shardRouting5)));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardRouting5, sameInstance(shardRouting1));
    assertThat(shardRouting6, sameInstance(shardRouting2));
    shardIterator = routingTable.index("test1").shard(0).shardsIt(3);
    assertThat(shardIterator.size(), equalTo(2));
    ShardRouting shardRouting7 = shardIterator.nextOrNull();
    assertThat(shardRouting7, notNullValue());
    ShardRouting shardRouting8 = shardIterator.nextOrNull();
    assertThat(shardRouting8, notNullValue());
    assertThat(shardRouting8, not(sameInstance(shardRouting7)));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardRouting7, sameInstance(shardRouting3));
    assertThat(shardRouting8, sameInstance(shardRouting4));
    shardIterator = routingTable.index("test1").shard(0).shardsIt(4);
    assertThat(shardIterator.size(), equalTo(2));
    ShardRouting shardRouting9 = shardIterator.nextOrNull();
    assertThat(shardRouting9, notNullValue());
    ShardRouting shardRouting10 = shardIterator.nextOrNull();
    assertThat(shardRouting10, notNullValue());
    assertThat(shardRouting10, not(sameInstance(shardRouting9)));
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardIterator.nextOrNull(), nullValue());
    assertThat(shardRouting9, sameInstance(shardRouting5));
    assertThat(shardRouting10, sameInstance(shardRouting6));
}

Example 8 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class CorruptedFileIT method testCorruptPrimaryNoReplica.

/**
     * Tests corruption that happens on a single shard when no replicas are present. We make sure that the primary stays unassigned
     * and all other replicas for the healthy shards happens
     */
public void testCorruptPrimaryNoReplica() throws ExecutionException, InterruptedException, IOException {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), // no checkindex - we corrupt shards on purpose
    false).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), // no translog based flush - it might change the .liv / segments.N files
    new ByteSizeValue(1, ByteSizeUnit.PB))));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);
    ShardRouting shardRouting = corruptRandomPrimaryFile();
    /*
         * we corrupted the primary shard - now lets make sure we never recover from it successfully
         */
    Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").build();
    client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
    client().admin().cluster().prepareReroute().get();
    boolean didClusterTurnRed = awaitBusy(() -> {
        ClusterHealthStatus test = client().admin().cluster().health(Requests.clusterHealthRequest("test")).actionGet().getStatus();
        return test == ClusterHealthStatus.RED;
    }, 5, // sometimes on slow nodes the replication / recovery is just dead slow
    TimeUnit.MINUTES);
    final ClusterHealthResponse response = client().admin().cluster().health(Requests.clusterHealthRequest("test")).get();
    if (response.getStatus() != ClusterHealthStatus.RED) {
        logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed);
        logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
    }
    assertThat(response.getStatus(), is(ClusterHealthStatus.RED));
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
    for (ShardIterator iterator : shardIterators) {
        ShardRouting routing;
        while ((routing = iterator.nextOrNull()) != null) {
            if (routing.getId() == shardRouting.getId()) {
                assertThat(routing.state(), equalTo(ShardRoutingState.UNASSIGNED));
            } else {
                assertThat(routing.state(), anyOf(equalTo(ShardRoutingState.RELOCATING), equalTo(ShardRoutingState.STARTED)));
            }
        }
    }
    final List<Path> files = listShardFiles(shardRouting);
    Path corruptedFile = null;
    for (Path file : files) {
        if (file.getFileName().toString().startsWith("corrupted_")) {
            corruptedFile = file;
            break;
        }
    }
    assertThat(corruptedFile, notNullValue());
}

Example 9 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class SuggestStatsIT method nodeIdsWithIndex.

private Set<String> nodeIdsWithIndex(String... indices) {
    ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
    GroupShardsIterator allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
    Set<String> nodes = new HashSet<>();
    for (ShardIterator shardIterator : allAssignedShardsGrouped) {
        for (ShardRouting routing : shardIterator.asUnordered()) {
            if (routing.active()) {
                nodes.add(routing.currentNodeId());
            }
        }
    }
    return nodes;
}

Example 10 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class TransportSearchAction method executeSearch.

private void executeSearch(SearchTask task, long startTimeInMillis, SearchRequest searchRequest, String[] localIndices, List<ShardIterator> remoteShardIterators, Function<String, Transport.Connection> remoteConnections, ClusterState clusterState, Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener) {
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
    // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
    // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
    // of just for the _search api
    final Index[] indices;
    if (localIndices.length == 0 && remoteShardIterators.size() > 0) {
        // don't search on _all if only remote indices were specified
        indices = Index.EMPTY_ARRAY;
    } else {
        indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(), startTimeInMillis, localIndices);
    }
    Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
    Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices());
    String[] concreteIndices = new String[indices.length];
    for (int i = 0; i < indices.length; i++) {
        concreteIndices[i] = indices[i].getName();
    }
    GroupShardsIterator localShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, searchRequest.preference());
    GroupShardsIterator shardIterators = mergeShardsIterators(localShardsIterator, remoteShardIterators);
    failIfOverShardCountLimit(clusterService, shardIterators.size());
    Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);
    // optimize search type for cases where there is only one shard group to search on
    if (shardIterators.size() == 1) {
        // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard
        searchRequest.searchType(QUERY_THEN_FETCH);
    }
    if (searchRequest.isSuggestOnly()) {
        // disable request cache if we have only suggest
        searchRequest.requestCache(false);
        switch(searchRequest.searchType()) {
            case DFS_QUERY_THEN_FETCH:
                // convert to Q_T_F if we have only suggest
                searchRequest.searchType(QUERY_THEN_FETCH);
                break;
        }
    }
    final DiscoveryNodes nodes = clusterState.nodes();
    Function<String, Transport.Connection> connectionLookup = (nodeId) -> {
        final DiscoveryNode discoveryNode = nodes.get(nodeId);
        final Transport.Connection connection;
        if (discoveryNode != null) {
            connection = searchTransportService.getConnection(discoveryNode);
        } else {
            connection = remoteConnections.apply(nodeId);
        }
        if (connection == null) {
            throw new IllegalStateException("no node found for id: " + nodeId);
        }
        return connection;
    };
    searchAsyncAction(task, searchRequest, shardIterators, startTimeInMillis, connectionLookup, clusterState.version(), Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, listener).start();
}