Examples with RemoteClusterService org.opensearch.transport.RemoteClusterService used on opensource projects

Example 1 with RemoteClusterService

use of org.opensearch.transport.RemoteClusterService in project OpenSearch by opensearch-project.

the class TransportSearchActionTests method testCollectSearchShards.

public void testCollectSearchShards() throws Exception {
    int numClusters = randomIntBetween(2, 10);
    DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
    Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
    Settings.Builder builder = Settings.builder();
    MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
    Settings settings = builder.build();
    try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
        service.start();
        service.acceptIncomingRequests();
        RemoteClusterService remoteClusterService = service.getRemoteClusterService();
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            AtomicInteger skippedClusters = new AtomicInteger();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters, map.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                assertTrue(map.containsKey(clusterAlias));
                ClusterSearchShardsResponse shardsResponse = map.get(clusterAlias);
                assertEquals(1, shardsResponse.getNodes().length);
            }
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicReference<Exception> failure = new AtomicReference<>();
            AtomicInteger skippedClusters = new AtomicInteger(0);
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), "index_not_found", null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(failure.get());
            assertThat(failure.get(), instanceOf(RemoteTransportException.class));
            RemoteTransportException remoteTransportException = (RemoteTransportException) failure.get();
            assertEquals(RestStatus.NOT_FOUND, remoteTransportException.status());
        }
        int numDisconnectedClusters = randomIntBetween(1, numClusters);
        Set<DiscoveryNode> disconnectedNodes = new HashSet<>(numDisconnectedClusters);
        Set<Integer> disconnectedNodesIndices = new HashSet<>(numDisconnectedClusters);
        while (disconnectedNodes.size() < numDisconnectedClusters) {
            int i = randomIntBetween(0, numClusters - 1);
            if (disconnectedNodes.add(nodes[i])) {
                assertTrue(disconnectedNodesIndices.add(i));
            }
        }
        CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters);
        RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() {

            @Override
            public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) {
                if (disconnectedNodes.remove(node)) {
                    disconnectedLatch.countDown();
                }
            }
        });
        for (DiscoveryNode disconnectedNode : disconnectedNodes) {
            service.addFailToSendNoConnectRule(disconnectedNode.getAddress());
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Exception> failure = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(failure.get());
            assertThat(failure.get(), instanceOf(RemoteTransportException.class));
            assertThat(failure.get().getMessage(), containsString("error while communicating with remote cluster ["));
            assertThat(failure.get().getCause(), instanceOf(NodeDisconnectedException.class));
        }
        // setting skip_unavailable to true for all the disconnected clusters will make the request succeed again
        for (int i : disconnectedNodesIndices) {
            RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true);
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters - disconnectedNodesIndices.size(), map.size());
            assertEquals(skippedClusters.get(), disconnectedNodesIndices.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                if (disconnectedNodesIndices.contains(i)) {
                    assertFalse(map.containsKey(clusterAlias));
                } else {
                    assertNotNull(map.get(clusterAlias));
                }
            }
        }
        // give transport service enough time to realize that the node is down, and to notify the connection listeners
        // so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next
        assertTrue(disconnectedLatch.await(5, TimeUnit.SECONDS));
        service.clearAllRules();
        if (randomBoolean()) {
            for (int i : disconnectedNodesIndices) {
                if (randomBoolean()) {
                    RemoteClusterServiceTests.updateSkipUnavailable(remoteClusterService, "remote" + i, true);
                }
            }
        }
        {
            final CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger skippedClusters = new AtomicInteger(0);
            AtomicReference<Map<String, ClusterSearchShardsResponse>> response = new AtomicReference<>();
            TransportSearchAction.collectSearchShards(IndicesOptions.lenientExpandOpen(), null, null, skippedClusters, remoteIndicesByCluster, remoteClusterService, threadPool, new LatchedActionListener<>(ActionListener.wrap(response::set, e -> fail("no failures expected")), latch));
            awaitLatch(latch, 5, TimeUnit.SECONDS);
            assertEquals(0, skippedClusters.get());
            assertNotNull(response.get());
            Map<String, ClusterSearchShardsResponse> map = response.get();
            assertEquals(numClusters, map.size());
            for (int i = 0; i < numClusters; i++) {
                String clusterAlias = "remote" + i;
                assertTrue(map.containsKey(clusterAlias));
                assertNotNull(map.get(clusterAlias));
            }
        }
        assertEquals(0, service.getConnectionManager().size());
    } finally {
        for (MockTransportService mockTransportService : mockTransportServices) {
            mockTransportService.close();
        }
    }
}

Example 2 with RemoteClusterService

use of org.opensearch.transport.RemoteClusterService in project OpenSearch by opensearch-project.

the class TransportSearchActionTests method testProcessRemoteShards.

public void testProcessRemoteShards() {
    try (TransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
        RemoteClusterService service = transportService.getRemoteClusterService();
        assertFalse(service.isCrossClusterSearchEnabled());
        Map<String, ClusterSearchShardsResponse> searchShardsResponseMap = new HashMap<>();
        DiscoveryNode[] nodes = new DiscoveryNode[] { new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT), new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT) };
        Map<String, AliasFilter> indicesAndAliases = new HashMap<>();
        indicesAndAliases.put("foo", new AliasFilter(new TermsQueryBuilder("foo", "bar"), "some_alias_for_foo", "some_other_foo_alias"));
        indicesAndAliases.put("bar", new AliasFilter(new MatchAllQueryBuilder(), Strings.EMPTY_ARRAY));
        ClusterSearchShardsGroup[] groups = new ClusterSearchShardsGroup[] { new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 0), new ShardRouting[] { TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("foo", 0, "node2", false, ShardRoutingState.STARTED) }), new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 1), new ShardRouting[] { TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("foo", 1, "node2", false, ShardRoutingState.STARTED) }), new ClusterSearchShardsGroup(new ShardId("bar", "bar_id", 0), new ShardRouting[] { TestShardRouting.newShardRouting("bar", 0, "node2", true, ShardRoutingState.STARTED), TestShardRouting.newShardRouting("bar", 0, "node1", false, ShardRoutingState.STARTED) }) };
        searchShardsResponseMap.put("test_cluster_1", new ClusterSearchShardsResponse(groups, nodes, indicesAndAliases));
        DiscoveryNode[] nodes2 = new DiscoveryNode[] { new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT) };
        ClusterSearchShardsGroup[] groups2 = new ClusterSearchShardsGroup[] { new ClusterSearchShardsGroup(new ShardId("xyz", "xyz_id", 0), new ShardRouting[] { TestShardRouting.newShardRouting("xyz", 0, "node3", true, ShardRoutingState.STARTED) }) };
        Map<String, AliasFilter> filter = new HashMap<>();
        filter.put("xyz", new AliasFilter(null, "some_alias_for_xyz"));
        searchShardsResponseMap.put("test_cluster_2", new ClusterSearchShardsResponse(groups2, nodes2, filter));
        Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
        remoteIndicesByCluster.put("test_cluster_1", new OriginalIndices(new String[] { "fo*", "ba*" }, SearchRequest.DEFAULT_INDICES_OPTIONS));
        remoteIndicesByCluster.put("test_cluster_2", new OriginalIndices(new String[] { "x*" }, SearchRequest.DEFAULT_INDICES_OPTIONS));
        Map<String, AliasFilter> remoteAliases = TransportSearchAction.getRemoteAliasFilters(searchShardsResponseMap);
        List<SearchShardIterator> iteratorList = TransportSearchAction.getRemoteShardsIterator(searchShardsResponseMap, remoteIndicesByCluster, remoteAliases);
        assertEquals(4, iteratorList.size());
        for (SearchShardIterator iterator : iteratorList) {
            if (iterator.shardId().getIndexName().endsWith("foo")) {
                assertArrayEquals(new String[] { "some_alias_for_foo", "some_other_foo_alias" }, iterator.getOriginalIndices().indices());
                assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1);
                assertEquals("test_cluster_1", iterator.getClusterAlias());
                assertEquals("foo", iterator.shardId().getIndexName());
                SearchShardTarget shard = iterator.nextOrNull();
                assertNotNull(shard);
                assertEquals(shard.getShardId().getIndexName(), "foo");
                shard = iterator.nextOrNull();
                assertNotNull(shard);
                assertEquals(shard.getShardId().getIndexName(), "foo");
                assertNull(iterator.nextOrNull());
            } else if (iterator.shardId().getIndexName().endsWith("bar")) {
                assertArrayEquals(new String[] { "bar" }, iterator.getOriginalIndices().indices());
                assertEquals(0, iterator.shardId().getId());
                assertEquals("test_cluster_1", iterator.getClusterAlias());
                assertEquals("bar", iterator.shardId().getIndexName());
                SearchShardTarget shard = iterator.nextOrNull();
                assertNotNull(shard);
                assertEquals(shard.getShardId().getIndexName(), "bar");
                shard = iterator.nextOrNull();
                assertNotNull(shard);
                assertEquals(shard.getShardId().getIndexName(), "bar");
                assertNull(iterator.nextOrNull());
            } else if (iterator.shardId().getIndexName().endsWith("xyz")) {
                assertArrayEquals(new String[] { "some_alias_for_xyz" }, iterator.getOriginalIndices().indices());
                assertEquals(0, iterator.shardId().getId());
                assertEquals("xyz", iterator.shardId().getIndexName());
                assertEquals("test_cluster_2", iterator.getClusterAlias());
                SearchShardTarget shard = iterator.nextOrNull();
                assertNotNull(shard);
                assertEquals(shard.getShardId().getIndexName(), "xyz");
                assertNull(iterator.nextOrNull());
            }
        }
        assertEquals(3, remoteAliases.size());
        assertTrue(remoteAliases.toString(), remoteAliases.containsKey("foo_id"));
        assertTrue(remoteAliases.toString(), remoteAliases.containsKey("bar_id"));
        assertTrue(remoteAliases.toString(), remoteAliases.containsKey("xyz_id"));
        assertEquals(new TermsQueryBuilder("foo", "bar"), remoteAliases.get("foo_id").getQueryBuilder());
        assertEquals(new MatchAllQueryBuilder(), remoteAliases.get("bar_id").getQueryBuilder());
        assertNull(remoteAliases.get("xyz_id").getQueryBuilder());
    }
}

Example 3 with RemoteClusterService

use of org.opensearch.transport.RemoteClusterService in project OpenSearch by opensearch-project.

the class TransportSearchActionTests method testCCSRemoteReduceMergeFails.

public void testCCSRemoteReduceMergeFails() throws Exception {
    int numClusters = randomIntBetween(2, 10);
    DiscoveryNode[] nodes = new DiscoveryNode[numClusters];
    Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
    Settings.Builder builder = Settings.builder();
    MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder);
    Settings settings = builder.build();
    boolean local = randomBoolean();
    OriginalIndices localIndices = local ? new OriginalIndices(new String[] { "index" }, SearchRequest.DEFAULT_INDICES_OPTIONS) : null;
    TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0);
    Function<Boolean, InternalAggregation.ReduceContext> reduceContext = finalReduce -> null;
    try (MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null)) {
        service.start();
        service.acceptIncomingRequests();
        RemoteClusterService remoteClusterService = service.getRemoteClusterService();
        SearchRequest searchRequest = new SearchRequest();
        searchRequest.preference("null_target");
        final CountDownLatch latch = new CountDownLatch(1);
        SetOnce<Tuple<SearchRequest, ActionListener<SearchResponse>>> setOnce = new SetOnce<>();
        AtomicReference<Exception> failure = new AtomicReference<>();
        LatchedActionListener<SearchResponse> listener = new LatchedActionListener<>(ActionListener.wrap(r -> fail("no response expected"), failure::set), latch);
        TransportSearchAction.ccsRemoteReduce(searchRequest, localIndices, remoteIndicesByCluster, timeProvider, emptyReduceContextBuilder(), remoteClusterService, threadPool, listener, (r, l) -> setOnce.set(Tuple.tuple(r, l)));
        if (localIndices == null) {
            assertNull(setOnce.get());
        } else {
            Tuple<SearchRequest, ActionListener<SearchResponse>> tuple = setOnce.get();
            assertEquals("", tuple.v1().getLocalClusterAlias());
            assertThat(tuple.v2(), instanceOf(TransportSearchAction.CCSActionListener.class));
            tuple.v2().onResponse(emptySearchResponse());
        }
        awaitLatch(latch, 5, TimeUnit.SECONDS);
        assertNotNull(failure.get());
        // the intention here is not to test that we throw NPE, rather to trigger a situation that makes
        // SearchResponseMerger#getMergedResponse fail unexpectedly and verify that the listener is properly notified with the NPE
        assertThat(failure.get(), instanceOf(NullPointerException.class));
        assertEquals(0, service.getConnectionManager().size());
    } finally {
        for (MockTransportService mockTransportService : mockTransportServices) {
            mockTransportService.close();
        }
    }
}

Example 4 with RemoteClusterService

use of org.opensearch.transport.RemoteClusterService in project OpenSearch by opensearch-project.

the class TransportSearchAction method executeRequest.

private void executeRequest(Task task, SearchRequest searchRequest, SearchAsyncActionProvider searchAsyncActionProvider, ActionListener<SearchResponse> listener) {
    final long relativeStartNanos = System.nanoTime();
    final SearchTimeProvider timeProvider = new SearchTimeProvider(searchRequest.getOrCreateAbsoluteStartMillis(), relativeStartNanos, System::nanoTime);
    ActionListener<SearchSourceBuilder> rewriteListener = ActionListener.wrap(source -> {
        if (source != searchRequest.source()) {
            // only set it if it changed - we don't allow null values to be set but it might be already null. this way we catch
            // situations when source is rewritten to null due to a bug
            searchRequest.source(source);
        }
        final ClusterState clusterState = clusterService.state();
        final SearchContextId searchContext;
        final Map<String, OriginalIndices> remoteClusterIndices;
        if (searchRequest.pointInTimeBuilder() != null) {
            searchContext = SearchContextId.decode(namedWriteableRegistry, searchRequest.pointInTimeBuilder().getId());
            remoteClusterIndices = getIndicesFromSearchContexts(searchContext, searchRequest.indicesOptions());
        } else {
            searchContext = null;
            remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState));
        }
        OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
        if (remoteClusterIndices.isEmpty()) {
            executeLocalSearch(task, timeProvider, searchRequest, localIndices, clusterState, listener, searchContext, searchAsyncActionProvider);
        } else {
            if (shouldMinimizeRoundtrips(searchRequest)) {
                ccsRemoteReduce(searchRequest, localIndices, remoteClusterIndices, timeProvider, searchService.aggReduceContextBuilder(searchRequest), remoteClusterService, threadPool, listener, (r, l) -> executeLocalSearch(task, timeProvider, r, localIndices, clusterState, l, searchContext, searchAsyncActionProvider));
            } else {
                AtomicInteger skippedClusters = new AtomicInteger(0);
                collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(), skippedClusters, remoteClusterIndices, remoteClusterService, threadPool, ActionListener.wrap(searchShardsResponses -> {
                    final BiFunction<String, String, DiscoveryNode> clusterNodeLookup = getRemoteClusterNodeLookup(searchShardsResponses);
                    final Map<String, AliasFilter> remoteAliasFilters;
                    final List<SearchShardIterator> remoteShardIterators;
                    if (searchContext != null) {
                        remoteAliasFilters = searchContext.aliasFilter();
                        remoteShardIterators = getRemoteShardsIteratorFromPointInTime(searchShardsResponses, searchContext, searchRequest.pointInTimeBuilder().getKeepAlive(), remoteClusterIndices);
                    } else {
                        remoteAliasFilters = getRemoteAliasFilters(searchShardsResponses);
                        remoteShardIterators = getRemoteShardsIterator(searchShardsResponses, remoteClusterIndices, remoteAliasFilters);
                    }
                    int localClusters = localIndices == null ? 0 : 1;
                    int totalClusters = remoteClusterIndices.size() + localClusters;
                    int successfulClusters = searchShardsResponses.size() + localClusters;
                    executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener, new SearchResponse.Clusters(totalClusters, successfulClusters, skippedClusters.get()), searchContext, searchAsyncActionProvider);
                }, listener::onFailure));
            }
        }
    }, listener::onFailure);
    if (searchRequest.source() == null) {
        rewriteListener.onResponse(searchRequest.source());
    } else {
        Rewriteable.rewriteAndFetch(searchRequest.source(), searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis), rewriteListener);
    }
}

Example 5 with RemoteClusterService

use of org.opensearch.transport.RemoteClusterService in project OpenSearch by opensearch-project.

the class SearchScrollAsyncAction method collectNodesAndRun.

/**
 * This method collects nodes from the remote clusters asynchronously if any of the scroll IDs references a remote cluster.
 * Otherwise the action listener will be invoked immediately with a function based on the given discovery nodes.
 */
static void collectNodesAndRun(final Iterable<SearchContextIdForNode> scrollIds, DiscoveryNodes nodes, SearchTransportService searchTransportService, ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
    Set<String> clusters = new HashSet<>();
    for (SearchContextIdForNode target : scrollIds) {
        if (target.getClusterAlias() != null) {
            clusters.add(target.getClusterAlias());
        }
    }
    if (clusters.isEmpty()) {
        // no remote clusters
        listener.onResponse((cluster, node) -> nodes.get(node));
    } else {
        RemoteClusterService remoteClusterService = searchTransportService.getRemoteClusterService();
        remoteClusterService.collectNodes(clusters, ActionListener.map(listener, nodeFunction -> (clusterAlias, node) -> clusterAlias == null ? nodes.get(node) : nodeFunction.apply(clusterAlias, node)));
    }
}