Examples with ShardIterator org.elasticsearch.cluster.routing.ShardIterator used on opensource projects

Example 21 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class RemoteClusterServiceTests method testProcessRemoteShards.

public void testProcessRemoteShards() throws IOException {
    try (RemoteClusterService service = new RemoteClusterService(Settings.EMPTY, null)) {
        assertFalse(service.isCrossClusterSearchEnabled());
        List<ShardIterator> iteratorList = new ArrayList<>();
        Map<String, ClusterSearchShardsResponse> searchShardsResponseMap = new HashMap<>();
        DiscoveryNode[] nodes = new DiscoveryNode[] { new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT), new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT) };
        Map<String, AliasFilter> indicesAndAliases = new HashMap<>();
        indicesAndAliases.put("foo", new AliasFilter(new TermsQueryBuilder("foo", "bar"), Strings.EMPTY_ARRAY));
        indicesAndAliases.put("bar", new AliasFilter(new MatchAllQueryBuilder(), Strings.EMPTY_ARRAY));
        ClusterSearchShardsGroup[] groups = new ClusterSearchShardsGroup[] { new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 0), new ShardRouting[] { TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("foo", 0, "node2", false, ShardRoutingState.STARTED) }), new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 1), new ShardRouting[] { TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("foo", 1, "node2", false, ShardRoutingState.STARTED) }), new ClusterSearchShardsGroup(new ShardId("bar", "bar_id", 0), new ShardRouting[] { TestShardRouting.newShardRouting("bar", 0, "node2", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("bar", 0, "node1", false, ShardRoutingState.STARTED) }) };
        searchShardsResponseMap.put("test_cluster_1", new ClusterSearchShardsResponse(groups, nodes, indicesAndAliases));
        Map<String, AliasFilter> remoteAliases = new HashMap<>();
        service.processRemoteShards(searchShardsResponseMap, iteratorList, remoteAliases);
        assertEquals(3, iteratorList.size());
        for (ShardIterator iterator : iteratorList) {
            if (iterator.shardId().getIndexName().endsWith("foo")) {
                assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1);
                assertEquals("test_cluster_1:foo", iterator.shardId().getIndexName());
                ShardRouting shardRouting = iterator.nextOrNull();
                assertNotNull(shardRouting);
                assertEquals(shardRouting.getIndexName(), "foo");
                shardRouting = iterator.nextOrNull();
                assertNotNull(shardRouting);
                assertEquals(shardRouting.getIndexName(), "foo");
                assertNull(iterator.nextOrNull());
            } else {
                assertEquals(0, iterator.shardId().getId());
                assertEquals("test_cluster_1:bar", iterator.shardId().getIndexName());
                ShardRouting shardRouting = iterator.nextOrNull();
                assertNotNull(shardRouting);
                assertEquals(shardRouting.getIndexName(), "bar");
                shardRouting = iterator.nextOrNull();
                assertNotNull(shardRouting);
                assertEquals(shardRouting.getIndexName(), "bar");
                assertNull(iterator.nextOrNull());
            }
        }
        assertEquals(2, remoteAliases.size());
        assertTrue(remoteAliases.toString(), remoteAliases.containsKey("foo_id"));
        assertTrue(remoteAliases.toString(), remoteAliases.containsKey("bar_id"));
        assertEquals(new TermsQueryBuilder("foo", "bar"), remoteAliases.get("foo_id").getQueryBuilder());
        assertEquals(new MatchAllQueryBuilder(), remoteAliases.get("bar_id").getQueryBuilder());
    }
}

Example 22 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class CorruptedFileIT method corruptRandomPrimaryFile.

private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    Index test = state.metaData().index("test").getIndex();
    GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
    List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
    ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
    ShardRouting shardRouting = shardIterator.nextOrNull();
    assertNotNull(shardRouting);
    assertTrue(shardRouting.primary());
    assertTrue(shardRouting.assignedToNode());
    String nodeId = shardRouting.currentNodeId();
    NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
    // treeset makes sure iteration order is deterministic
    Set<Path> files = new TreeSet<>();
    for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
        String path = info.getPath();
        Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
        if (Files.exists(file)) {
            // multi data path might only have one path in use
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                for (Path item : stream) {
                    if (Files.isRegularFile(item) && "write.lock".equals(item.getFileName().toString()) == false) {
                        if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
                            files.add(item);
                        }
                    }
                }
            }
        }
    }
    pruneOldDeleteGenerations(files);
    CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
    return shardRouting;
}

Example 23 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class CorruptedTranslogIT method corruptRandomTranslogFiles.

private void corruptRandomTranslogFiles() throws IOException {
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
    final Index test = state.metaData().index("test").getIndex();
    List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
    ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
    ShardRouting shardRouting = shardIterator.nextOrNull();
    assertNotNull(shardRouting);
    assertTrue(shardRouting.primary());
    assertTrue(shardRouting.assignedToNode());
    String nodeId = shardRouting.currentNodeId();
    NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
    // treeset makes sure iteration order is deterministic
    Set<Path> files = new TreeSet<>();
    for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
        String path = fsPath.getPath();
        final String relativeDataLocationPath = "indices/" + test.getUUID() + "/" + Integer.toString(shardRouting.getId()) + "/translog";
        Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
        if (Files.exists(file)) {
            logger.info("--> path: {}", file);
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                for (Path item : stream) {
                    logger.info("--> File: {}", item);
                    if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) {
                        files.add(item);
                    }
                }
            }
        }
    }
    Path fileToCorrupt = null;
    if (!files.isEmpty()) {
        int corruptions = randomIntBetween(5, 20);
        for (int i = 0; i < corruptions; i++) {
            fileToCorrupt = RandomPicks.randomFrom(random(), files);
            try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
                // read
                raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
                long filePointer = raf.position();
                ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
                raf.read(bb);
                bb.flip();
                // corrupt
                byte oldValue = bb.get(0);
                byte newValue = (byte) (oldValue + 1);
                bb.put(0, newValue);
                // rewrite
                raf.position(filePointer);
                raf.write(bb);
                logger.info("--> corrupting file {} --  flipping at position {} from {} to {} file: {}", fileToCorrupt, filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue), fileToCorrupt);
            }
        }
    }
    assertThat("no file corrupted", fileToCorrupt, notNullValue());
}

Example 24 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class ShardFailedClusterStateTaskExecutorTests method createExistingShards.

private List<ShardStateAction.ShardEntry> createExistingShards(ClusterState currentState, String reason) {
    List<ShardRouting> shards = new ArrayList<>();
    GroupShardsIterator shardGroups = currentState.routingTable().allAssignedShardsGrouped(new String[] { INDEX }, true);
    for (ShardIterator shardIt : shardGroups) {
        for (ShardRouting shard : shardIt.asUnordered()) {
            shards.add(shard);
        }
    }
    List<ShardRouting> failures = randomSubsetOf(randomIntBetween(1, 1 + shards.size() / 4), shards.toArray(new ShardRouting[0]));
    String indexUUID = metaData.index(INDEX).getIndexUUID();
    int numberOfTasks = randomIntBetween(failures.size(), 2 * failures.size());
    List<ShardRouting> shardsToFail = new ArrayList<>(numberOfTasks);
    for (int i = 0; i < numberOfTasks; i++) {
        shardsToFail.add(randomFrom(failures));
    }
    return toTasks(currentState, shardsToFail, indexUUID, reason);
}

Example 25 with ShardIterator

use of org.elasticsearch.cluster.routing.ShardIterator in project elasticsearch by elastic.

the class RoutingIteratorTests method testShardsAndPreferNodeRouting.

public void testShardsAndPreferNodeRouting() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).localNodeId("node1")).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(1));
    //check node preference, first without preference to see they switch
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    String firstRoundNodeId = shardIterators.iterator().next().nextOrNull().currentNodeId();
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId)));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1,node2");
    assertThat(shardIterators.size(), equalTo(1));
    Iterator<ShardIterator> iterator = shardIterators.iterator();
    final ShardIterator it = iterator.next();
    assertThat(it.shardId().id(), equalTo(0));
    final String firstNodeId = it.nextOrNull().currentNodeId();
    assertThat(firstNodeId, anyOf(equalTo("node1"), equalTo("node2")));
    if ("node1".equals(firstNodeId)) {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node2"));
    } else {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node1"));
    }
}