Examples with ByteSizeValue org.opensearch.common.unit.ByteSizeValue used on opensource projects

Example 1 with ByteSizeValue

use of org.opensearch.common.unit.ByteSizeValue in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorWaitOnClose.

public void testBulkProcessorWaitOnClose() throws Exception {
    BulkProcessorTestListener listener = new BulkProcessorTestListener();
    int numDocs = randomIntBetween(10, 100);
    BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10), RandomPicks.randomFrom(random(), ByteSizeUnit.values()))).build();
    MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
    assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
    if (randomBoolean()) {
        // check if we can call it multiple times
        if (randomBoolean()) {
            assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
        } else {
            processor.close();
        }
    }
    assertThat(listener.beforeCounts.get(), greaterThanOrEqualTo(1));
    assertThat(listener.afterCounts.get(), greaterThanOrEqualTo(1));
    for (Throwable bulkFailure : listener.bulkFailures) {
        logger.error("bulk failure", bulkFailure);
    }
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertResponseItems(listener.bulkItems, numDocs);
    assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
}

Example 2 with ByteSizeValue

use of org.opensearch.common.unit.ByteSizeValue in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testThatBulkProcessorCountIsCorrect.

public void testThatBulkProcessorCountIsCorrect() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
    }
}

Example 3 with ByteSizeValue

use of org.opensearch.common.unit.ByteSizeValue in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testGlobalParametersAndBulkProcessor.

public void testGlobalParametersAndBulkProcessor() throws Exception {
    createIndexWithMultipleShards("test");
    createFieldAddingPipleine("pipeline_id", "fieldNameXYZ", "valueXYZ");
    int numDocs = randomIntBetween(10, 10);
    {
        final CountDownLatch latch = new CountDownLatch(1);
        BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
        try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).setGlobalIndex("test").setGlobalRouting("routing").setGlobalPipeline("pipeline_id").build()) {
            indexDocs(processor, numDocs, null, "test", "pipeline_id");
            latch.await();
            assertThat(listener.beforeCounts.get(), equalTo(1));
            assertThat(listener.afterCounts.get(), equalTo(1));
            assertThat(listener.bulkFailures.size(), equalTo(0));
            assertResponseItems(listener.bulkItems, numDocs);
            Iterable<SearchHit> hits = searchAll(new SearchRequest("test").routing("routing"));
            assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
            assertThat(hits, containsInAnyOrder(expectedIds(numDocs)));
        }
    }
}

Example 4 with ByteSizeValue

use of org.opensearch.common.unit.ByteSizeValue in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorFlush.

public void testBulkProcessorFlush() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 10)).setBulkActions(numDocs + randomIntBetween(1, 100)).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
        assertThat(latch.await(randomInt(500), TimeUnit.MILLISECONDS), equalTo(false));
        // we really need an explicit flush as none of the bulk thresholds was reached
        processor.flush();
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
    }
}

Example 5 with ByteSizeValue

use of org.opensearch.common.unit.ByteSizeValue in project OpenSearch by opensearch-project.

the class IndicesClientIT method testRollover.

public void testRollover() throws IOException {
    highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT);
    RolloverRequest rolloverRequest = new RolloverRequest("alias", "test_new");
    rolloverRequest.addMaxIndexDocsCondition(1);
    {
        RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync);
        assertFalse(rolloverResponse.isRolledOver());
        assertFalse(rolloverResponse.isDryRun());
        Map<String, Boolean> conditionStatus = rolloverResponse.getConditionStatus();
        assertEquals(1, conditionStatus.size());
        assertFalse(conditionStatus.get("[max_docs: 1]"));
        assertEquals("test", rolloverResponse.getOldIndex());
        assertEquals("test_new", rolloverResponse.getNewIndex());
    }
    highLevelClient().index(new IndexRequest("test").id("1").source("field", "value"), RequestOptions.DEFAULT);
    highLevelClient().index(new IndexRequest("test").id("2").source("field", "value").setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT);
    // without the refresh the rollover may not happen as the number of docs seen may be off
    {
        rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1));
        rolloverRequest.dryRun(true);
        RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync);
        assertFalse(rolloverResponse.isRolledOver());
        assertTrue(rolloverResponse.isDryRun());
        Map<String, Boolean> conditionStatus = rolloverResponse.getConditionStatus();
        assertEquals(2, conditionStatus.size());
        assertTrue(conditionStatus.get("[max_docs: 1]"));
        assertTrue(conditionStatus.get("[max_age: 1ms]"));
        assertEquals("test", rolloverResponse.getOldIndex());
        assertEquals("test_new", rolloverResponse.getNewIndex());
    }
    {
        String mappings = "{\"properties\":{\"field2\":{\"type\":\"keyword\"}}}";
        rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON);
        rolloverRequest.dryRun(false);
        rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB));
        RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync);
        assertTrue(rolloverResponse.isRolledOver());
        assertFalse(rolloverResponse.isDryRun());
        Map<String, Boolean> conditionStatus = rolloverResponse.getConditionStatus();
        assertEquals(3, conditionStatus.size());
        assertTrue(conditionStatus.get("[max_docs: 1]"));
        assertTrue(conditionStatus.get("[max_age: 1ms]"));
        assertFalse(conditionStatus.get("[max_size: 1mb]"));
        assertEquals("test", rolloverResponse.getOldIndex());
        assertEquals("test_new", rolloverResponse.getNewIndex());
    }
}