Examples with MultiGetRequestBuilder org.elasticsearch.action.get.MultiGetRequestBuilder used on opensource projects

Example 11 with MultiGetRequestBuilder

use of org.elasticsearch.action.get.MultiGetRequestBuilder in project elasticsearch by elastic.

the class BulkProcessorIT method testThatBulkProcessorCountIsCorrect.

public void testThatBulkProcessorCountIsCorrect() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setName("foo").setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
    }
}

Example 12 with MultiGetRequestBuilder

use of org.elasticsearch.action.get.MultiGetRequestBuilder in project elasticsearch by elastic.

the class BulkProcessorIT method testBulkProcessorConcurrentRequests.

public void testBulkProcessorConcurrentRequests() throws Exception {
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 7);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    MultiGetRequestBuilder multiGetRequestBuilder;
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(numDocs));
    Set<String> ids = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
        assertThat(bulkItemResponse.getIndex(), equalTo("test"));
        assertThat(bulkItemResponse.getType(), equalTo("test"));
        //with concurrent requests > 1 we can't rely on the order of the bulk requests
        assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
        //we do want to check that we don't get duplicate ids back
        assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
}

Example 13 with MultiGetRequestBuilder

use of org.elasticsearch.action.get.MultiGetRequestBuilder in project elasticsearch by elastic.

the class BulkProcessorIT method testBulkProcessorConcurrentRequestsReadOnlyIndex.

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
    createIndex("test-ro");
    assertAcked(client().admin().indices().prepareUpdateSettings("test-ro").setSettings(Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true)));
    ensureGreen();
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 10);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    int testDocs = 0;
    int testReadOnlyDocs = 0;
    MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet();
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        for (int i = 1; i <= numDocs; i++) {
            if (randomBoolean()) {
                testDocs++;
                processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs));
            } else {
                testReadOnlyDocs++;
                processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
            }
        }
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
    Set<String> ids = new HashSet<>();
    Set<String> readOnlyIds = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
        assertThat(bulkItemResponse.getType(), equalTo("test"));
        if (bulkItemResponse.getIndex().equals("test")) {
            assertThat(bulkItemResponse.isFailed(), equalTo(false));
            //with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
            //we do want to check that we don't get duplicate ids back
            assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
        } else {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            //with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
            //we do want to check that we don't get duplicate ids back
            assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
        }
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), testDocs);
}