Examples with MultiGetRequestBuilder org.opensearch.action.get.MultiGetRequestBuilder used on opensource projects

Example 1 with MultiGetRequestBuilder

use of org.opensearch.action.get.MultiGetRequestBuilder in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testThatBulkProcessorCountIsCorrect.

public void testThatBulkProcessorCountIsCorrect() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = BulkProcessor.builder(client()::bulk, listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
    }
}

Example 2 with MultiGetRequestBuilder

use of org.opensearch.action.get.MultiGetRequestBuilder in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorFlush.

public void testBulkProcessorFlush() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = BulkProcessor.builder(client()::bulk, listener).setConcurrentRequests(randomIntBetween(0, 10)).setBulkActions(numDocs + randomIntBetween(1, 100)).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        assertThat(latch.await(randomInt(500), TimeUnit.MILLISECONDS), equalTo(false));
        // we really need an explicit flush as none of the bulk thresholds was reached
        processor.flush();
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
    }
}

Example 3 with MultiGetRequestBuilder

use of org.opensearch.action.get.MultiGetRequestBuilder in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorConcurrentRequestsReadOnlyIndex.

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
    createIndex("test-ro");
    assertAcked(client().admin().indices().prepareUpdateSettings("test-ro").setSettings(Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true)));
    ensureGreen();
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 10);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    int testDocs = 0;
    int testReadOnlyDocs = 0;
    MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet();
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    try (BulkProcessor processor = BulkProcessor.builder(client()::bulk, listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        for (int i = 1; i <= numDocs; i++) {
            if (randomBoolean()) {
                testDocs++;
                processor.add(new IndexRequest("test").id(Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                multiGetRequestBuilder.add("test", Integer.toString(testDocs));
            } else {
                testReadOnlyDocs++;
                processor.add(new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
            }
        }
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
    Set<String> ids = new HashSet<>();
    Set<String> readOnlyIds = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
        if (bulkItemResponse.getIndex().equals("test")) {
            assertThat(bulkItemResponse.isFailed(), equalTo(false));
            // with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
            // we do want to check that we don't get duplicate ids back
            assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
        } else {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            // with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
            // we do want to check that we don't get duplicate ids back
            assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
        }
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), testDocs);
}

Example 4 with MultiGetRequestBuilder

use of org.opensearch.action.get.MultiGetRequestBuilder in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorConcurrentRequests.

public void testBulkProcessorConcurrentRequests() throws Exception {
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 7);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    MultiGetRequestBuilder multiGetRequestBuilder;
    try (BulkProcessor processor = BulkProcessor.builder(client()::bulk, listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(numDocs));
    Set<String> ids = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
        assertThat(bulkItemResponse.getIndex(), equalTo("test"));
        // with concurrent requests > 1 we can't rely on the order of the bulk requests
        assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
        // we do want to check that we don't get duplicate ids back
        assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
}

Example 5 with MultiGetRequestBuilder

use of org.opensearch.action.get.MultiGetRequestBuilder in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorWaitOnClose.

public void testBulkProcessorWaitOnClose() throws Exception {
    BulkProcessorTestListener listener = new BulkProcessorTestListener();
    int numDocs = randomIntBetween(10, 100);
    BulkProcessor processor = BulkProcessor.builder(client()::bulk, listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10), RandomPicks.randomFrom(random(), ByteSizeUnit.values()))).build();
    MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
    assertThat(processor.isOpen(), is(true));
    assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
    if (randomBoolean()) {
        // check if we can call it multiple times
        if (randomBoolean()) {
            assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
        } else {
            processor.close();
        }
    }
    assertThat(processor.isOpen(), is(false));
    assertThat(listener.beforeCounts.get(), greaterThanOrEqualTo(1));
    assertThat(listener.afterCounts.get(), greaterThanOrEqualTo(1));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertResponseItems(listener.bulkItems, numDocs);
    assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
}