Examples with ShardIterator org.elasticsearch.cluster.routing.ShardIterator used on opensource projects

Example 16 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class RemoteClusterService method processRemoteShards.

Function<String, Transport.Connection> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses, List<ShardIterator> remoteShardIterators, Map<String, AliasFilter> aliasFilterMap) {
    Map<String, Supplier<Transport.Connection>> nodeToCluster = new HashMap<>();
    for (Map.Entry<String, ClusterSearchShardsResponse> entry : searchShardsResponses.entrySet()) {
        String clusterName = entry.getKey();
        ClusterSearchShardsResponse searchShardsResponse = entry.getValue();
        for (DiscoveryNode remoteNode : searchShardsResponse.getNodes()) {
            nodeToCluster.put(remoteNode.getId(), () -> getConnection(remoteNode, clusterName));
        }
        Map<String, AliasFilter> indicesAndFilters = searchShardsResponse.getIndicesAndFilters();
        for (ClusterSearchShardsGroup clusterSearchShardsGroup : searchShardsResponse.getGroups()) {
            //add the cluster name to the remote index names for indices disambiguation
            //this ends up in the hits returned with the search response
            ShardId shardId = clusterSearchShardsGroup.getShardId();
            Index remoteIndex = shardId.getIndex();
            Index index = new Index(clusterName + REMOTE_CLUSTER_INDEX_SEPARATOR + remoteIndex.getName(), remoteIndex.getUUID());
            ShardIterator shardIterator = new PlainShardIterator(new ShardId(index, shardId.getId()), Arrays.asList(clusterSearchShardsGroup.getShards()));
            remoteShardIterators.add(shardIterator);
            AliasFilter aliasFilter;
            if (indicesAndFilters == null) {
                aliasFilter = new AliasFilter(null, Strings.EMPTY_ARRAY);
            } else {
                aliasFilter = indicesAndFilters.get(shardId.getIndexName());
                assert aliasFilter != null;
            }
            // here we have to map the filters to the UUID since from now on we use the uuid for the lookup
            aliasFilterMap.put(remoteIndex.getUUID(), aliasFilter);
        }
    }
    return (nodeId) -> {
        Supplier<Transport.Connection> supplier = nodeToCluster.get(nodeId);
        if (supplier == null) {
            throw new IllegalArgumentException("unknown remote node: " + nodeId);
        }
        return supplier.get();
    };
}

Example 17 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class InitialSearchPhase method run.

@Override
public final void run() throws IOException {
    int shardIndex = -1;
    for (final ShardIterator shardIt : shardsIts) {
        shardIndex++;
        final ShardRouting shard = shardIt.nextOrNull();
        if (shard != null) {
            performPhaseOnShard(shardIndex, shardIt, shard);
        } else {
            // really, no shards active in this group
            onShardFailure(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
        }
    }
}

Example 18 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class TransportClusterSearchShardsAction method masterOperation.

@Override
protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener<ClusterSearchShardsResponse> listener) {
    ClusterState clusterState = clusterService.state();
    String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
    Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
    Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
    for (String index : concreteIndices) {
        AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, request.indices());
        indicesAndFilters.put(index, aliasFilter);
    }
    Set<String> nodeIds = new HashSet<>();
    GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
    ShardRouting shard;
    ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];
    int currentGroup = 0;
    for (ShardIterator shardIt : groupShardsIterator) {
        ShardId shardId = shardIt.shardId();
        ShardRouting[] shardRoutings = new ShardRouting[shardIt.size()];
        int currentShard = 0;
        shardIt.reset();
        while ((shard = shardIt.nextOrNull()) != null) {
            shardRoutings[currentShard++] = shard;
            nodeIds.add(shard.currentNodeId());
        }
        groupResponses[currentGroup++] = new ClusterSearchShardsGroup(shardId, shardRoutings);
    }
    DiscoveryNode[] nodes = new DiscoveryNode[nodeIds.size()];
    int currentNode = 0;
    for (String nodeId : nodeIds) {
        nodes[currentNode++] = clusterState.getNodes().get(nodeId);
    }
    listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes, indicesAndFilters));
}

Example 19 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class SearchAsyncActionTests method getShardsIter.

private GroupShardsIterator getShardsIter(String index, int numShards, boolean doReplicas, DiscoveryNode primaryNode, DiscoveryNode replicaNode) {
    ArrayList<ShardIterator> list = new ArrayList<>();
    for (int i = 0; i < numShards; i++) {
        ArrayList<ShardRouting> started = new ArrayList<>();
        ArrayList<ShardRouting> initializing = new ArrayList<>();
        ArrayList<ShardRouting> unassigned = new ArrayList<>();
        ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
        routing = routing.initialize(primaryNode.getId(), i + "p", 0);
        routing.started();
        started.add(routing);
        if (doReplicas) {
            routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), false, RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
            if (replicaNode != null) {
                routing = routing.initialize(replicaNode.getId(), i + "r", 0);
                if (randomBoolean()) {
                    routing.started();
                    started.add(routing);
                } else {
                    initializing.add(routing);
                }
            } else {
                // unused yet
                unassigned.add(routing);
            }
        }
        Collections.shuffle(started, random());
        started.addAll(initializing);
        list.add(new PlainShardIterator(new ShardId(new Index(index, "_na_"), i), started));
    }
    return new GroupShardsIterator(list);
}

Example 20 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class SearchAsyncActionTests method testFanOutAndCollect.

public void testFanOutAndCollect() throws InterruptedException {
    SearchRequest request = new SearchRequest();
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<TestSearchResponse> response = new AtomicReference<>();
    ActionListener<SearchResponse> responseListener = new ActionListener<SearchResponse>() {

        @Override
        public void onResponse(SearchResponse searchResponse) {
            response.set((TestSearchResponse) searchResponse);
        }

        @Override
        public void onFailure(Exception e) {
            logger.warn("test failed", e);
            fail(e.getMessage());
        }
    };
    DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
    Map<DiscoveryNode, Set<Long>> nodeToContextMap = new HashMap<>();
    AtomicInteger contextIdGenerator = new AtomicInteger(0);
    GroupShardsIterator shardsIter = getShardsIter("idx", randomIntBetween(1, 10), randomBoolean(), primaryNode, replicaNode);
    AtomicInteger numFreedContext = new AtomicInteger();
    SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, Collections.singleton(RemoteClusterService.REMOTE_CLUSTERS_SEEDS)), null) {

        @Override
        public void sendFreeContext(Transport.Connection connection, long contextId, SearchRequest request) {
            numFreedContext.incrementAndGet();
            assertTrue(nodeToContextMap.containsKey(connection.getNode()));
            assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
        }
    };
    Map<String, Transport.Connection> lookup = new HashMap<>();
    lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
    lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
    Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
    AbstractSearchAsyncAction asyncAction = new AbstractSearchAsyncAction<TestSearchPhaseResult>("test", logger, transportService, lookup::get, aliasFilters, Collections.emptyMap(), null, request, responseListener, shardsIter, 0, 0, null, new InitialSearchPhase.SearchPhaseResults<>(shardsIter.size())) {

        TestSearchResponse response = new TestSearchResponse();

        @Override
        protected void executePhaseOnShard(ShardIterator shardIt, ShardRouting shard, ActionListener<TestSearchPhaseResult> listener) {
            assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
            Transport.Connection connection = getConnection(shard.currentNodeId());
            TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(), connection.getNode());
            Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> new HashSet<>());
            ids.add(testSearchPhaseResult.id);
            if (randomBoolean()) {
                listener.onResponse(testSearchPhaseResult);
            } else {
                new Thread(() -> listener.onResponse(testSearchPhaseResult)).start();
            }
        }

        @Override
        protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
            return new SearchPhase("test") {

                @Override
                public void run() throws IOException {
                    for (int i = 0; i < results.getNumShards(); i++) {
                        TestSearchPhaseResult result = results.results.get(i);
                        assertEquals(result.node.getId(), result.shardTarget().getNodeId());
                        sendReleaseSearchContext(result.id(), new MockConnection(result.node));
                    }
                    responseListener.onResponse(response);
                    latch.countDown();
                }
            };
        }
    };
    asyncAction.start();
    latch.await();
    assertNotNull(response.get());
    assertFalse(nodeToContextMap.isEmpty());
    assertTrue(nodeToContextMap.toString(), nodeToContextMap.containsKey(primaryNode) || nodeToContextMap.containsKey(replicaNode));
    assertEquals(shardsIter.size(), numFreedContext.get());
    if (nodeToContextMap.containsKey(primaryNode)) {
        assertTrue(nodeToContextMap.get(primaryNode).toString(), nodeToContextMap.get(primaryNode).isEmpty());
    } else {
        assertTrue(nodeToContextMap.get(replicaNode).toString(), nodeToContextMap.get(replicaNode).isEmpty());
    }
}