Examples with ByteSizeValue org.elasticsearch.common.unit.ByteSizeValue used on opensource projects

Example 86 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch by elastic.

the class BulkProcessorIT method testThatBulkProcessorCountIsCorrect.

public void testThatBulkProcessorCountIsCorrect() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setName("foo").setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
    }
}

Example 87 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch by elastic.

the class IndexRequestTests method testToStringSizeLimit.

public void testToStringSizeLimit() throws UnsupportedEncodingException {
    IndexRequest request = new IndexRequest("index", "type");
    String source = "{\"name\":\"value\"}";
    request.source(source, XContentType.JSON);
    assertEquals("index {[index][type][null], source[" + source + "]}", request.toString());
    source = "{\"name\":\"" + randomUnicodeOfLength(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING) + "\"}";
    request.source(source, XContentType.JSON);
    int actualBytes = source.getBytes("UTF-8").length;
    assertEquals("index {[index][type][null], source[n/a, actual length: [" + new ByteSizeValue(actualBytes).toString() + "], max length: " + new ByteSizeValue(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING).toString() + "]}", request.toString());
}

Example 88 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch by elastic.

the class BulkProcessorIT method testBulkProcessorConcurrentRequests.

public void testBulkProcessorConcurrentRequests() throws Exception {
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 7);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    MultiGetRequestBuilder multiGetRequestBuilder;
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(numDocs));
    Set<String> ids = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
        assertThat(bulkItemResponse.getIndex(), equalTo("test"));
        assertThat(bulkItemResponse.getType(), equalTo("test"));
        //with concurrent requests > 1 we can't rely on the order of the bulk requests
        assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
        //we do want to check that we don't get duplicate ids back
        assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
}

Example 89 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch by elastic.

the class BulkProcessorIT method testBulkProcessorConcurrentRequestsReadOnlyIndex.

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
    createIndex("test-ro");
    assertAcked(client().admin().indices().prepareUpdateSettings("test-ro").setSettings(Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true)));
    ensureGreen();
    int bulkActions = randomIntBetween(10, 100);
    int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
    int concurrentRequests = randomIntBetween(0, 10);
    int expectedBulkActions = numDocs / bulkActions;
    final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
    int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
    final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
    int testDocs = 0;
    int testReadOnlyDocs = 0;
    MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet();
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
    try (BulkProcessor processor = BulkProcessor.builder(client(), listener).setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        for (int i = 1; i <= numDocs; i++) {
            if (randomBoolean()) {
                testDocs++;
                processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs));
            } else {
                testReadOnlyDocs++;
                processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
            }
        }
    }
    closeLatch.await();
    assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
    Set<String> ids = new HashSet<>();
    Set<String> readOnlyIds = new HashSet<>();
    for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
        assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
        assertThat(bulkItemResponse.getType(), equalTo("test"));
        if (bulkItemResponse.getIndex().equals("test")) {
            assertThat(bulkItemResponse.isFailed(), equalTo(false));
            //with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
            //we do want to check that we don't get duplicate ids back
            assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
        } else {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            //with concurrent requests > 1 we can't rely on the order of the bulk requests
            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
            //we do want to check that we don't get duplicate ids back
            assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
        }
    }
    assertMultiGetResponse(multiGetRequestBuilder.get(), testDocs);
}

Example 90 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch by elastic.

the class DiskThresholdSettingsTests method testDefaults.

public void testDefaults() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(Settings.EMPTY, nss);
    ByteSizeValue zeroBytes = ByteSizeValue.parseBytesSizeValue("0b", "test");
    assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdHigh());
    assertEquals(10.0D, diskThresholdSettings.getFreeDiskThresholdHigh(), 0.0D);
    assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdLow());
    assertEquals(15.0D, diskThresholdSettings.getFreeDiskThresholdLow(), 0.0D);
    assertEquals(60L, diskThresholdSettings.getRerouteInterval().seconds());
    assertTrue(diskThresholdSettings.isEnabled());
    assertTrue(diskThresholdSettings.includeRelocations());
}