Examples with MultiGetRequest org.opensearch.action.get.MultiGetRequest used on opensource projects

Example 11 with MultiGetRequest

use of org.opensearch.action.get.MultiGetRequest in project OpenSearch by opensearch-project.

the class BulkProcessorRetryIT method executeBulkRejectionLoad.

private void executeBulkRejectionLoad(BackoffPolicy backoffPolicy, boolean rejectedExecutionExpected) throws Exception {
    final CorrelatingBackoffPolicy internalPolicy = new CorrelatingBackoffPolicy(backoffPolicy);
    final int numberOfAsyncOps = randomIntBetween(600, 700);
    final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps);
    final Set<Object> responses = Collections.newSetFromMap(new ConcurrentHashMap<>());
    BulkProcessor bulkProcessor = initBulkProcessorBuilder(new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            internalPolicy.logResponse(response);
            responses.add(response);
            latch.countDown();
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            internalPolicy.logResponse(failure);
            responses.add(failure);
            latch.countDown();
        }
    }).setBulkActions(1).setConcurrentRequests(randomIntBetween(0, 100)).setBackoffPolicy(internalPolicy).build();
    MultiGetRequest multiGetRequest = indexDocs(bulkProcessor, numberOfAsyncOps);
    latch.await(10, TimeUnit.SECONDS);
    bulkProcessor.close();
    assertEquals(responses.size(), numberOfAsyncOps);
    boolean rejectedAfterAllRetries = false;
    for (Object response : responses) {
        if (response instanceof BulkResponse) {
            BulkResponse bulkResponse = (BulkResponse) response;
            for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
                if (bulkItemResponse.isFailed()) {
                    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                    if (failure.getStatus() == RestStatus.TOO_MANY_REQUESTS) {
                        if (rejectedExecutionExpected == false) {
                            assertRetriedCorrectly(internalPolicy, bulkResponse, failure.getCause());
                            rejectedAfterAllRetries = true;
                        }
                    } else {
                        throw new AssertionError("Unexpected failure with status: " + failure.getStatus());
                    }
                }
            }
        } else {
            if (response instanceof RemoteTransportException && ((RemoteTransportException) response).status() == RestStatus.TOO_MANY_REQUESTS) {
                if (rejectedExecutionExpected == false) {
                    assertRetriedCorrectly(internalPolicy, response, ((Throwable) response).getCause());
                    rejectedAfterAllRetries = true;
                }
            // ignored, we exceeded the write queue size when dispatching the initial bulk request
            } else {
                Throwable t = (Throwable) response;
                // we're not expecting any other errors
                throw new AssertionError("Unexpected failure", t);
            }
        }
    }
    highLevelClient().indices().refresh(new RefreshRequest(), RequestOptions.DEFAULT);
    int multiGetResponsesCount = highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT).getResponses().length;
    if (rejectedExecutionExpected) {
        assertThat(multiGetResponsesCount, lessThanOrEqualTo(numberOfAsyncOps));
    } else if (rejectedAfterAllRetries) {
        assertThat(multiGetResponsesCount, lessThan(numberOfAsyncOps));
    } else {
        assertThat(multiGetResponsesCount, equalTo(numberOfAsyncOps));
    }
}

Example 12 with MultiGetRequest

use of org.opensearch.action.get.MultiGetRequest in project OpenSearch by opensearch-project.

the class BulkProcessorIT method indexDocs.

private MultiGetRequest indexDocs(BulkProcessor processor, int numDocs, String localIndex, String globalIndex, String globalPipeline) throws Exception {
    MultiGetRequest multiGetRequest = new MultiGetRequest();
    for (int i = 1; i <= numDocs; i++) {
        if (randomBoolean()) {
            processor.add(new IndexRequest(localIndex).id(Integer.toString(i)).source(XContentType.JSON, "field", randomRealisticUnicodeOfLengthBetween(1, 30)));
        } else {
            BytesArray data = bytesBulkRequest(localIndex, i);
            processor.add(data, globalIndex, globalPipeline, XContentType.JSON);
        }
        multiGetRequest.add(localIndex, Integer.toString(i));
    }
    return multiGetRequest;
}

Example 13 with MultiGetRequest

use of org.opensearch.action.get.MultiGetRequest in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorConcurrentRequestsReadOnlyIndex.

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
    Request request = new Request("PUT", "/test-ro");
    request.setJsonEntity("{\n" + "    \"settings\" : {\n" + "        \"index\" : {\n" + "            \"blocks.write\" : true\n" + "        }\n" + "    }\n" + "    \n" + "}");
    Response response = client().performRequest(request);
    assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 10);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    int testDocs = 0;
    int testReadOnlyDocs = 0;
    MultiGetRequest multiGetRequest = new MultiGetRequest();
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        for (int i = 1; i <= numDocs; i++) {
            // let's make sure we get at least 1 item in the MultiGetRequest regardless of the randomising roulette
            if (randomBoolean() || multiGetRequest.getItems().size() == 0) {
                testDocs++;
                processor.add(new IndexRequest("test").id(Integer.toString(testDocs)).source(XContentType.JSON, "field", "value"));
                multiGetRequest.add("test", Integer.toString(testDocs));
            } else {
                testReadOnlyDocs++;
                processor.add(new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)).source(XContentType.JSON, "field", "value"));
            }
        }
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
    Set<String> ids = new HashSet<>();
    Set<String> readOnlyIds = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
        if (bulkItemResponse.getIndex().equals("test")) {
            assertThat(bulkItemResponse.isFailed(), equalTo(false));
            // with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
            // we do want to check that we don't get duplicate ids back
            assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
        } else {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            // with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
            // we do want to check that we don't get duplicate ids back
            assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
        }
    }
    assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), testDocs);
}

Example 14 with MultiGetRequest

use of org.opensearch.action.get.MultiGetRequest in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorConcurrentRequests.

public void testBulkProcessorConcurrentRequests() throws Exception {
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 7);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    MultiGetRequest multiGetRequest;
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        multiGetRequest = indexDocs(processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(numDocs));
    Set<String> ids = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
        assertThat(bulkItemResponse.getIndex(), equalTo("test"));
        // with concurrent requests > 1 we can't rely on the order of the bulk requests
        assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
        // we do want to check that we don't get duplicate ids back
        assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
    }
    assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
}

Example 15 with MultiGetRequest

use of org.opensearch.action.get.MultiGetRequest in project OpenSearch by opensearch-project.

the class RequestConverters method multiGet.

static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_mget");
    Params parameters = new Params();
    parameters.withPreference(multiGetRequest.preference());
    parameters.withRealtime(multiGetRequest.realtime());
    parameters.withRefresh(multiGetRequest.refresh());
    request.addParameters(parameters.asMap());
    request.setEntity(createEntity(multiGetRequest, REQUEST_BODY_CONTENT_TYPE));
    return request;
}