Examples with Scroll org.opensearch.search.Scroll used on opensource projects

Example 1 with Scroll

use of org.opensearch.search.Scroll in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testMultiSearch.

public void testMultiSearch() throws IOException {
    int numberOfSearchRequests = randomIntBetween(0, 32);
    MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
    for (int i = 0; i < numberOfSearchRequests; i++) {
        SearchRequest searchRequest = randomSearchRequest(() -> {
            // No need to return a very complex SearchSourceBuilder here, that is tested
            // elsewhere
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.from(randomInt(10));
            searchSourceBuilder.size(randomIntBetween(20, 100));
            return searchSourceBuilder;
        });
        // scroll is not supported in the current msearch api, so unset it:
        searchRequest.scroll((Scroll) null);
        // only expand_wildcards, ignore_unavailable and allow_no_indices can be
        // specified from msearch api, so unset other options:
        IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
        IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
        searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(), msearchDefault.expandWildcardsHidden(), msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), msearchDefault.ignoreThrottled()));
        multiSearchRequest.add(searchRequest);
    }
    Map<String, String> expectedParams = new HashMap<>();
    expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
    if (randomBoolean()) {
        multiSearchRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8));
        expectedParams.put("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
    }
    Request request = RequestConverters.multiSearch(multiSearchRequest);
    assertEquals("/_msearch", request.getEndpoint());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    assertEquals(expectedParams, request.getParameters());
    List<SearchRequest> requests = new ArrayList<>();
    CheckedBiConsumer<SearchRequest, XContentParser, IOException> consumer = (searchRequest, p) -> {
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(p, false);
        if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) {
            searchRequest.source(searchSourceBuilder);
        }
        requests.add(searchRequest);
    };
    MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())), REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, xContentRegistry(), true, deprecationLogger);
    assertEquals(requests, multiSearchRequest.requests());
}

Example 2 with Scroll

use of org.opensearch.search.Scroll in project OpenSearch by opensearch-project.

the class SearchScrollAsyncActionTests method testNodeNotAvailable.

public void testNodeNotAvailable() throws InterruptedException {
    ParsedScrollId scrollId = getParsedScrollId(new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)), new SearchContextIdForNode(null, "node2", new ShardSearchContextId("", 2)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 17)), new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 0)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 0)));
    // node2 is not available
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT)).build();
    AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
    SearchScrollRequest request = new SearchScrollRequest();
    request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicInteger movedCounter = new AtomicInteger(0);
    SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, dummyListener(), null, request, null) {

        @Override
        protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest, SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener) {
            try {
                assertNotEquals("node2 is not available", "node2", connection.getNode().getId());
            } catch (NullPointerException e) {
                logger.warn(e);
            }
            new Thread(() -> {
                SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                searchActionListener.onResponse(testSearchPhaseResult);
            }).start();
        }

        @Override
        protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            return new SearchAsyncActionTests.MockConnection(node);
        }

        @Override
        protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
            assertEquals(1, movedCounter.incrementAndGet());
            return new SearchPhase("test") {

                @Override
                public void run() throws IOException {
                    latch.countDown();
                }
            };
        }

        @Override
        protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
            results.setOnce(shardId, result);
        }
    };
    action.run();
    latch.await();
    ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
    assertEquals(1, shardSearchFailures.length);
    assertEquals("IllegalStateException[node [node2] is not available]", shardSearchFailures[0].reason());
    SearchContextIdForNode[] context = scrollId.getContext();
    for (int i = 0; i < results.length(); i++) {
        if (context[i].getNode().equals("node2")) {
            assertNull(results.get(i));
        } else {
            assertNotNull(results.get(i));
            assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
            assertEquals(context[i].getNode(), results.get(i).node.getId());
        }
    }
}

Example 3 with Scroll

use of org.opensearch.search.Scroll in project OpenSearch by opensearch-project.

the class SearchScrollAsyncActionTests method testFailNextPhase.

public void testFailNextPhase() throws InterruptedException {
    ParsedScrollId scrollId = getParsedScrollId(new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)), new SearchContextIdForNode(null, "node2", new ShardSearchContextId("a", 2)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("b", 17)), new SearchContextIdForNode(null, "node1", new ShardSearchContextId("c", 0)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("d", 0)));
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT)).build();
    AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
    SearchScrollRequest request = new SearchScrollRequest();
    request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicInteger movedCounter = new AtomicInteger(0);
    ActionListener<SearchResponse> listener = new ActionListener<SearchResponse>() {

        @Override
        public void onResponse(SearchResponse o) {
            try {
                fail("got a result");
            } finally {
                latch.countDown();
            }
        }

        @Override
        public void onFailure(Exception e) {
            try {
                assertTrue(e instanceof SearchPhaseExecutionException);
                SearchPhaseExecutionException ex = (SearchPhaseExecutionException) e;
                assertEquals("BOOM", ex.getCause().getMessage());
                assertEquals("TEST_PHASE", ex.getPhaseName());
                assertEquals("Phase failed", ex.getMessage());
            } finally {
                latch.countDown();
            }
        }
    };
    SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, listener, null, request, null) {

        @Override
        protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest, SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener) {
            new Thread(() -> {
                SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                searchActionListener.onResponse(testSearchPhaseResult);
            }).start();
        }

        @Override
        protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            return new SearchAsyncActionTests.MockConnection(node);
        }

        @Override
        protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
            assertEquals(1, movedCounter.incrementAndGet());
            return new SearchPhase("TEST_PHASE") {

                @Override
                public void run() throws IOException {
                    throw new IllegalArgumentException("BOOM");
                }
            };
        }

        @Override
        protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
            results.setOnce(shardId, result);
        }
    };
    action.run();
    latch.await();
    ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
    assertEquals(0, shardSearchFailures.length);
    SearchContextIdForNode[] context = scrollId.getContext();
    for (int i = 0; i < results.length(); i++) {
        assertNotNull(results.get(i));
        assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
        assertEquals(context[i].getNode(), results.get(i).node.getId());
    }
}

Example 4 with Scroll

use of org.opensearch.search.Scroll in project OpenSearch by opensearch-project.

the class SearchScrollAsyncActionTests method testSendRequestsToNodes.

public void testSendRequestsToNodes() throws InterruptedException {
    ParsedScrollId scrollId = getParsedScrollId(new SearchContextIdForNode(null, "node1", new ShardSearchContextId(UUIDs.randomBase64UUID(), 1)), new SearchContextIdForNode(null, "node2", new ShardSearchContextId(UUIDs.randomBase64UUID(), 2)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId(UUIDs.randomBase64UUID(), 17)), new SearchContextIdForNode(null, "node1", new ShardSearchContextId(UUIDs.randomBase64UUID(), 0)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId(UUIDs.randomBase64UUID(), 0)));
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT)).build();
    AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
    SearchScrollRequest request = new SearchScrollRequest();
    request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicInteger movedCounter = new AtomicInteger(0);
    SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, dummyListener(), null, request, null) {

        @Override
        protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest, SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener) {
            new Thread(() -> {
                SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult = new SearchAsyncActionTests.TestSearchPhaseResult(internalRequest.contextId(), connection.getNode());
                testSearchPhaseResult.setSearchShardTarget(new SearchShardTarget(connection.getNode().getId(), new ShardId("test", "_na_", 1), null, OriginalIndices.NONE));
                searchActionListener.onResponse(testSearchPhaseResult);
            }).start();
        }

        @Override
        protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            return new SearchAsyncActionTests.MockConnection(node);
        }

        @Override
        protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
            assertEquals(1, movedCounter.incrementAndGet());
            return new SearchPhase("test") {

                @Override
                public void run() throws IOException {
                    latch.countDown();
                }
            };
        }

        @Override
        protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
            results.setOnce(shardId, result);
        }
    };
    action.run();
    latch.await();
    ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
    assertEquals(0, shardSearchFailures.length);
    SearchContextIdForNode[] context = scrollId.getContext();
    for (int i = 0; i < results.length(); i++) {
        assertNotNull(results.get(i));
        assertEquals(context[i].getSearchContextId(), results.get(i).getContextId());
        assertEquals(context[i].getNode(), results.get(i).node.getId());
    }
}

Example 5 with Scroll

use of org.opensearch.search.Scroll in project OpenSearch by opensearch-project.

the class SearchScrollAsyncActionTests method testAllShardsFailed.

public void testAllShardsFailed() throws InterruptedException {
    ParsedScrollId scrollId = getParsedScrollId(new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 1)), new SearchContextIdForNode(null, "node2", new ShardSearchContextId("", 2)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 17)), new SearchContextIdForNode(null, "node1", new ShardSearchContextId("", 0)), new SearchContextIdForNode(null, "node3", new ShardSearchContextId("", 0)));
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT)).add(new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT)).build();
    AtomicArray<SearchAsyncActionTests.TestSearchPhaseResult> results = new AtomicArray<>(scrollId.getContext().length);
    SearchScrollRequest request = new SearchScrollRequest();
    request.scroll(new Scroll(TimeValue.timeValueMinutes(1)));
    CountDownLatch latch = new CountDownLatch(1);
    ActionListener<SearchResponse> listener = new ActionListener<SearchResponse>() {

        @Override
        public void onResponse(SearchResponse o) {
            try {
                fail("got a result");
            } finally {
                latch.countDown();
            }
        }

        @Override
        public void onFailure(Exception e) {
            try {
                assertTrue(e instanceof SearchPhaseExecutionException);
                SearchPhaseExecutionException ex = (SearchPhaseExecutionException) e;
                assertEquals("BOOM on shard", ex.getCause().getMessage());
                assertEquals("query", ex.getPhaseName());
                assertEquals("all shards failed", ex.getMessage());
            } finally {
                latch.countDown();
            }
        }
    };
    SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult> action = new SearchScrollAsyncAction<SearchAsyncActionTests.TestSearchPhaseResult>(scrollId, logger, discoveryNodes, listener, null, request, null) {

        @Override
        protected void executeInitialPhase(Transport.Connection connection, InternalScrollSearchRequest internalRequest, SearchActionListener<SearchAsyncActionTests.TestSearchPhaseResult> searchActionListener) {
            new Thread(() -> searchActionListener.onFailure(new IllegalArgumentException("BOOM on shard"))).start();
        }

        @Override
        protected Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            return new SearchAsyncActionTests.MockConnection(node);
        }

        @Override
        protected SearchPhase moveToNextPhase(BiFunction<String, String, DiscoveryNode> clusterNodeLookup) {
            fail("don't move all shards failed");
            return null;
        }

        @Override
        protected void onFirstPhaseResult(int shardId, SearchAsyncActionTests.TestSearchPhaseResult result) {
            results.setOnce(shardId, result);
        }
    };
    action.run();
    latch.await();
    SearchContextIdForNode[] context = scrollId.getContext();
    ShardSearchFailure[] shardSearchFailures = action.buildShardFailures();
    assertEquals(context.length, shardSearchFailures.length);
    assertEquals("IllegalArgumentException[BOOM on shard]", shardSearchFailures[0].reason());
    for (int i = 0; i < results.length(); i++) {
        assertNull(results.get(i));
    }
}