Examples with OriginalIndices org.opensearch.action.OriginalIndices used on opensource projects

Example 1 with OriginalIndices

use of org.opensearch.action.OriginalIndices in project OpenSearch by opensearch-project.

the class TransportSearchAction method getRemoteShardsIterator.

static List<SearchShardIterator> getRemoteShardsIterator(Map<String, ClusterSearchShardsResponse> searchShardsResponses, Map<String, OriginalIndices> remoteIndicesByCluster, Map<String, AliasFilter> aliasFilterMap) {
    final List<SearchShardIterator> remoteShardIterators = new ArrayList<>();
    for (Map.Entry<String, ClusterSearchShardsResponse> entry : searchShardsResponses.entrySet()) {
        for (ClusterSearchShardsGroup clusterSearchShardsGroup : entry.getValue().getGroups()) {
            // add the cluster name to the remote index names for indices disambiguation
            // this ends up in the hits returned with the search response
            ShardId shardId = clusterSearchShardsGroup.getShardId();
            AliasFilter aliasFilter = aliasFilterMap.get(shardId.getIndex().getUUID());
            String[] aliases = aliasFilter.getAliases();
            String clusterAlias = entry.getKey();
            String[] finalIndices = aliases.length == 0 ? new String[] { shardId.getIndexName() } : aliases;
            final OriginalIndices originalIndices = remoteIndicesByCluster.get(clusterAlias);
            assert originalIndices != null : "original indices are null for clusterAlias: " + clusterAlias;
            SearchShardIterator shardIterator = new SearchShardIterator(clusterAlias, shardId, Arrays.asList(clusterSearchShardsGroup.getShards()), new OriginalIndices(finalIndices, originalIndices.indicesOptions()));
            remoteShardIterators.add(shardIterator);
        }
    }
    return remoteShardIterators;
}

Example 2 with OriginalIndices

use of org.opensearch.action.OriginalIndices in project OpenSearch by opensearch-project.

the class TransportSearchAction method collectSearchShards.

static void collectSearchShards(IndicesOptions indicesOptions, String preference, String routing, AtomicInteger skippedClusters, Map<String, OriginalIndices> remoteIndicesByCluster, RemoteClusterService remoteClusterService, ThreadPool threadPool, ActionListener<Map<String, ClusterSearchShardsResponse>> listener) {
    final CountDown responsesCountDown = new CountDown(remoteIndicesByCluster.size());
    final Map<String, ClusterSearchShardsResponse> searchShardsResponses = new ConcurrentHashMap<>();
    final AtomicReference<Exception> exceptions = new AtomicReference<>();
    for (Map.Entry<String, OriginalIndices> entry : remoteIndicesByCluster.entrySet()) {
        final String clusterAlias = entry.getKey();
        boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias);
        Client clusterClient = remoteClusterService.getRemoteClusterClient(threadPool, clusterAlias);
        final String[] indices = entry.getValue().indices();
        ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(indices).indicesOptions(indicesOptions).local(true).preference(preference).routing(routing);
        clusterClient.admin().cluster().searchShards(searchShardsRequest, new CCSActionListener<ClusterSearchShardsResponse, Map<String, ClusterSearchShardsResponse>>(clusterAlias, skipUnavailable, responsesCountDown, skippedClusters, exceptions, listener) {

            @Override
            void innerOnResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) {
                searchShardsResponses.put(clusterAlias, clusterSearchShardsResponse);
            }

            @Override
            Map<String, ClusterSearchShardsResponse> createFinalResponse() {
                return searchShardsResponses;
            }
        });
    }
}

Example 3 with OriginalIndices

use of org.opensearch.action.OriginalIndices in project OpenSearch by opensearch-project.

the class RemoteClusterService method groupIndices.

public Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String[] indices, Predicate<String> indexExists) {
    Map<String, OriginalIndices> originalIndicesMap = new HashMap<>();
    if (isCrossClusterSearchEnabled()) {
        final Map<String, List<String>> groupedIndices = groupClusterIndices(getRemoteClusterNames(), indices, indexExists);
        if (groupedIndices.isEmpty()) {
            // search on _all in the local cluster if neither local indices nor remote indices were specified
            originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(Strings.EMPTY_ARRAY, indicesOptions));
        } else {
            for (Map.Entry<String, List<String>> entry : groupedIndices.entrySet()) {
                String clusterAlias = entry.getKey();
                List<String> originalIndices = entry.getValue();
                originalIndicesMap.put(clusterAlias, new OriginalIndices(originalIndices.toArray(new String[0]), indicesOptions));
            }
        }
    } else {
        originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(indices, indicesOptions));
    }
    return originalIndicesMap;
}

Example 4 with OriginalIndices

use of org.opensearch.action.OriginalIndices in project OpenSearch by opensearch-project.

the class TransportSearchActionTests method testCollectSearchShards.

public void testCollectSearchShards() throws Exception {
    int numClusters = randomIntBetween(2, 10);
    DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
    Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
    Settings.Builder builder = Settings.builder();
    MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
    Settings settings = builder.build();
    try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
        service.start();
        service.acceptIncomingRequests();
        RemoteClusterService remoteClusterService = service.getRemoteClusterService();
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            AtomicInteger skippedClusters = new AtomicInteger();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters, map.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                assertTrue(map.containsKey(clusterAlias));
                ClusterSearchShardsResponse shardsResponse = map.get(clusterAlias);
                assertEquals(1, shardsResponse.getNodes().length);
            }
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicReference<Exception> failure = new AtomicReference<>();
            AtomicInteger skippedClusters = new AtomicInteger(0);
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found", null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(failure.get());
            assertThat(failure.get(), instanceOf(RemoteTransportException.class));
            RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get();
            assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status());
        }
        int numDisconnectedClusters = randomIntBetween(1, numClusters);
        Set<DiscoveryNode> disconnectedNodes = new HashSet<>(numDisconnectedClusters);
        Set<Integer> disconnectedNodesIndices = new HashSet<>(numDisconnectedClusters);
        while (disconnectedNodes.size() < numDisconnectedClusters) {
            int i = randomIntBetween(0, numClusters - 1);
            if (disconnectedNodes.add(nodes[i])) {
                assertTrue(disconnectedNodesIndices.add(i));
            }
        }
        CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters);
        RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() {

            @Override
            public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) {
                if (disconnectedNodes.remove(node)) {
                    disconnectedLatch.countDown();
                }
            }
        });
        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
            service.addFailToSendNoConnectRule(disconnectedNode.getAddress());
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Exception> failure = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(failure.get());
            assertThat(failure.get(), instanceOf(RemoteTransportException.class));
            assertThat(failure.get().getMessage(), containsString("error while communicating with remote cluster ["));
            assertThat(failure.get().getCause(), instanceOf(NodeDisconnectedException.class));
        }
        // setting skip_unavailable to true for all the disconnected clusters will make the request succeed again
        for (int i : disconnectedNodesIndices) {
            RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true);
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters - disconnectedNodesIndices.size(), map.size());
            assertEquals(skippedClusters.get(), disconnectedNodesIndices.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                if (disconnectedNodesIndices.contains(i)) {
                    assertFalse(map.containsKey(clusterAlias));
                } else {
                    assertNotNull(map.get(clusterAlias));
                }
            }
        }
        // give transport service enough time to realize that the node is down, and to notify the connection listeners
        // so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next
        assertTrue(disconnectedLatch.await(5, TimeUnit.SECONDS));
        service.clearAllRules();
        if (randomBoolean()) {
            for (int i : disconnectedNodesIndices) {
                if (randomBoolean()) {
                    RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true);
                }
            }
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters, map.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                assertTrue(map.containsKey(clusterAlias));
                assertNotNull(map.get(clusterAlias));
            }
        }
        assertEquals(0, service.getConnectionManager().size());
    } finally {
        for (MockTransportService mockTransportService : mockTransportServices) {
            mockTransportService.close();
        }
    }
}

Example 5 with OriginalIndices

use of org.opensearch.action.OriginalIndices in project OpenSearch by opensearch-project.

the class TransportSearchActionTests method startTransport.

private MockTransportService[] startTransport(int numClusters, DiscoveryNode[] nodes, Map<String, OriginalIndices> remoteIndices, Settings.Builder settingsBuilder) {
    MockTransportService[] mockTransportServices = new MockTransportService[numClusters];
    for (int i = 0; i < numClusters; i++) {
        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
        MockTransportService remoteSeedTransport = RemoteClusterConnectionTests.startTransport("node_remote" + i, knownNodes, Version.CURRENT, threadPool);
        mockTransportServices[i] = remoteSeedTransport;
        DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalDiscoNode();
        knownNodes.add(remoteSeedNode);
        nodes[i] = remoteSeedNode;
        settingsBuilder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString());
        remoteIndices.put("remote" + i, new OriginalIndices(new String[] { "index" }, IndicesOptions.lenientExpandOpen()));
    }
    return mockTransportServices;
}