Examples with GroupShardsIterator org.elasticsearch.cluster.routing.GroupShardsIterator used on opensource projects

Example 11 with GroupShardsIterator

use of org.elasticsearch.cluster.routing.GroupShardsIterator in project elasticsearch by elastic.

the class TransportClusterSearchShardsAction method masterOperation.

@Override
protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener<ClusterSearchShardsResponse> listener) {
    ClusterState clusterState = clusterService.state();
    String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
    Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
    Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
    for (String index : concreteIndices) {
        AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, request.indices());
        indicesAndFilters.put(index, aliasFilter);
    }
    Set<String> nodeIds = new HashSet<>();
    GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
    ShardRouting shard;
    ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];
    int currentGroup = 0;
    for (ShardIterator shardIt : groupShardsIterator) {
        ShardId shardId = shardIt.shardId();
        ShardRouting[] shardRoutings = new ShardRouting[shardIt.size()];
        int currentShard = 0;
        shardIt.reset();
        while ((shard = shardIt.nextOrNull()) != null) {
            shardRoutings[currentShard++] = shard;
            nodeIds.add(shard.currentNodeId());
        }
        groupResponses[currentGroup++] = new ClusterSearchShardsGroup(shardId, shardRoutings);
    }
    DiscoveryNode[] nodes = new DiscoveryNode[nodeIds.size()];
    int currentNode = 0;
    for (String nodeId : nodeIds) {
        nodes[currentNode++] = clusterState.getNodes().get(nodeId);
    }
    listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes, indicesAndFilters));
}

Example 12 with GroupShardsIterator

use of org.elasticsearch.cluster.routing.GroupShardsIterator in project elasticsearch by elastic.

the class SearchAsyncActionTests method getShardsIter.

private GroupShardsIterator getShardsIter(String index, int numShards, boolean doReplicas, DiscoveryNode primaryNode, DiscoveryNode replicaNode) {
    ArrayList<ShardIterator> list = new ArrayList<>();
    for (int i = 0; i < numShards; i++) {
        ArrayList<ShardRouting> started = new ArrayList<>();
        ArrayList<ShardRouting> initializing = new ArrayList<>();
        ArrayList<ShardRouting> unassigned = new ArrayList<>();
        ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
        routing = routing.initialize(primaryNode.getId(), i + "p", 0);
        routing.started();
        started.add(routing);
        if (doReplicas) {
            routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), false, RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
            if (replicaNode != null) {
                routing = routing.initialize(replicaNode.getId(), i + "r", 0);
                if (randomBoolean()) {
                    routing.started();
                    started.add(routing);
                } else {
                    initializing.add(routing);
                }
            } else {
                // unused yet
                unassigned.add(routing);
            }
        }
        Collections.shuffle(started, random());
        started.addAll(initializing);
        list.add(new PlainShardIterator(new ShardId(new Index(index, "_na_"), i), started));
    }
    return new GroupShardsIterator(list);
}

Example 13 with GroupShardsIterator

use of org.elasticsearch.cluster.routing.GroupShardsIterator in project elasticsearch by elastic.

the class SearchAsyncActionTests method testFanOutAndCollect.

public void testFanOutAndCollect() throws InterruptedException {
    SearchRequest request = new SearchRequest();
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<TestSearchResponse> response = new AtomicReference<>();
    ActionListener<SearchResponse> responseListener = new ActionListener<SearchResponse>() {

        @Override
        public void onResponse(SearchResponse searchResponse) {
            response.set((TestSearchResponse) searchResponse);
        }

        @Override
        public void onFailure(Exception e) {
            logger.warn("test failed", e);
            fail(e.getMessage());
        }
    };
    DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
    Map<DiscoveryNode, Set<Long>> nodeToContextMap = new HashMap<>();
    AtomicInteger contextIdGenerator = new AtomicInteger(0);
    GroupShardsIterator shardsIter = getShardsIter("idx", randomIntBetween(1, 10), randomBoolean(), primaryNode, replicaNode);
    AtomicInteger numFreedContext = new AtomicInteger();
    SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, Collections.singleton(RemoteClusterService.REMOTE_CLUSTERS_SEEDS)), null) {

        @Override
        public void sendFreeContext(Transport.Connection connection, long contextId, SearchRequest request) {
            numFreedContext.incrementAndGet();
            assertTrue(nodeToContextMap.containsKey(connection.getNode()));
            assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
        }
    };
    Map<String, Transport.Connection> lookup = new HashMap<>();
    lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
    lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
    Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
    AbstractSearchAsyncAction asyncAction = new AbstractSearchAsyncAction<TestSearchPhaseResult>("test", logger, transportService, lookup::get, aliasFilters, Collections.emptyMap(), null, request, responseListener, shardsIter, 0, 0, null, new InitialSearchPhase.SearchPhaseResults<>(shardsIter.size())) {

        TestSearchResponse response = new TestSearchResponse();

        @Override
        protected void executePhaseOnShard(ShardIterator shardIt, ShardRouting shard, ActionListener<TestSearchPhaseResult> listener) {
            assertTrue("shard: " + shard.shardId() + " has been queried twice", response.queried.add(shard.shardId()));
            Transport.Connection connection = getConnection(shard.currentNodeId());
            TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(), connection.getNode());
            Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> new HashSet<>());
            ids.add(testSearchPhaseResult.id);
            if (randomBoolean()) {
                listener.onResponse(testSearchPhaseResult);
            } else {
                new Thread(() -> listener.onResponse(testSearchPhaseResult)).start();
            }
        }

        @Override
        protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
            return new SearchPhase("test") {

                @Override
                public void run() throws IOException {
                    for (int i = 0; i < results.getNumShards(); i++) {
                        TestSearchPhaseResult result = results.results.get(i);
                        assertEquals(result.node.getId(), result.shardTarget().getNodeId());
                        sendReleaseSearchContext(result.id(), new MockConnection(result.node));
                    }
                    responseListener.onResponse(response);
                    latch.countDown();
                }
            };
        }
    };
    asyncAction.start();
    latch.await();
    assertNotNull(response.get());
    assertFalse(nodeToContextMap.isEmpty());
    assertTrue(nodeToContextMap.toString(), nodeToContextMap.containsKey(primaryNode) || nodeToContextMap.containsKey(replicaNode));
    assertEquals(shardsIter.size(), numFreedContext.get());
    if (nodeToContextMap.containsKey(primaryNode)) {
        assertTrue(nodeToContextMap.get(primaryNode).toString(), nodeToContextMap.get(primaryNode).isEmpty());
    } else {
        assertTrue(nodeToContextMap.get(replicaNode).toString(), nodeToContextMap.get(replicaNode).isEmpty());
    }
}

Example 14 with GroupShardsIterator

use of org.elasticsearch.cluster.routing.GroupShardsIterator in project elasticsearch by elastic.

the class CorruptedFileIT method corruptRandomPrimaryFile.

private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    Index test = state.metaData().index("test").getIndex();
    GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
    List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
    ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
    ShardRouting shardRouting = shardIterator.nextOrNull();
    assertNotNull(shardRouting);
    assertTrue(shardRouting.primary());
    assertTrue(shardRouting.assignedToNode());
    String nodeId = shardRouting.currentNodeId();
    NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
    // treeset makes sure iteration order is deterministic
    Set<Path> files = new TreeSet<>();
    for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
        String path = info.getPath();
        Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
        if (Files.exists(file)) {
            // multi data path might only have one path in use
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                for (Path item : stream) {
                    if (Files.isRegularFile(item) && "write.lock".equals(item.getFileName().toString()) == false) {
                        if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
                            files.add(item);
                        }
                    }
                }
            }
        }
    }
    pruneOldDeleteGenerations(files);
    CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
    return shardRouting;
}

Example 15 with GroupShardsIterator

use of org.elasticsearch.cluster.routing.GroupShardsIterator in project elasticsearch by elastic.

the class CorruptedTranslogIT method corruptRandomTranslogFiles.

private void corruptRandomTranslogFiles() throws IOException {
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
    final Index test = state.metaData().index("test").getIndex();
    List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
    ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
    ShardRouting shardRouting = shardIterator.nextOrNull();
    assertNotNull(shardRouting);
    assertTrue(shardRouting.primary());
    assertTrue(shardRouting.assignedToNode());
    String nodeId = shardRouting.currentNodeId();
    NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
    // treeset makes sure iteration order is deterministic
    Set<Path> files = new TreeSet<>();
    for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
        String path = fsPath.getPath();
        final String relativeDataLocationPath = "indices/" + test.getUUID() + "/" + Integer.toString(shardRouting.getId()) + "/translog";
        Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
        if (Files.exists(file)) {
            logger.info("--> path: {}", file);
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                for (Path item : stream) {
                    logger.info("--> File: {}", item);
                    if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) {
                        files.add(item);
                    }
                }
            }
        }
    }
    Path fileToCorrupt = null;
    if (!files.isEmpty()) {
        int corruptions = randomIntBetween(5, 20);
        for (int i = 0; i < corruptions; i++) {
            fileToCorrupt = RandomPicks.randomFrom(random(), files);
            try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
                // read
                raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
                long filePointer = raf.position();
                ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
                raf.read(bb);
                bb.flip();
                // corrupt
                byte oldValue = bb.get(0);
                byte newValue = (byte) (oldValue + 1);
                bb.put(0, newValue);
                // rewrite
                raf.position(filePointer);
                raf.write(bb);
                logger.info("--> corrupting file {} --  flipping at position {} from {} to {} file: {}", fileToCorrupt, filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue), fileToCorrupt);
            }
        }
    }
    assertThat("no file corrupted", fileToCorrupt, notNullValue());
}