Examples with BulkProcessor org.opensearch.action.bulk.BulkProcessor used on opensource projects

Example 1 with BulkProcessor

use of org.opensearch.action.bulk.BulkProcessor in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

@Override
public ReindexResponse reindex() {
    if (currentReindexing == null) {
        currentReindexing = new ReindexResponse(docType);
    }
    final ReindexResponse resp = currentReindexing;
    if (resp.getStatus() == processing) {
        return resp;
    }
    // Create a new index.
    final String indexName;
    try {
        indexName = newIndex();
    } catch (final Throwable t) {
        return Response.error(resp, t);
    }
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final var clResp = sch.getClient();
    if (!clResp.isOk()) {
        return Response.fromResponse(resp, clResp);
    }
    final var cl = clResp.getEntity();
    final BulkListener listener = new BulkListener();
    final BulkProcessor.Builder builder = BulkProcessor.builder((request, bulkListener) -> cl.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
    final BulkProcessor bulkProcessor = builder.setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    /*
    SearchResponse scrollResp = cl.prepareSearch(targetIndex)
                                  .setSearchType(SearchType.SCAN)
                                  .setScroll(tv)
                                  .setQuery(qb)
                                  .setSize(batchSize)
                                  .execute()
                                  .actionGet(); //100 hits per shard will be returned for each scroll
*/
    final SearchSourceBuilder ssb = new SearchSourceBuilder().size(batchSize).query(qb);
    final SearchRequest sr = new SearchRequest(targetIndex).source(ssb).scroll(tv);
    // Switch to new index
    targetIndex = indexName;
    try {
        SearchResponse scrollResp = cl.search(sr, RequestOptions.DEFAULT);
        if (scrollResp.status() != RestStatus.OK) {
            if (debug()) {
                debug("Search returned status " + scrollResp.status());
            }
        }
        // Scroll until no hits are returned
        while (true) {
            for (final SearchHit hit : scrollResp.getHits().getHits()) {
                resp.incProcessed();
                if ((resp.getProcessed() % 250) == 0) {
                    info("processed " + docType + ": " + resp.getProcessed());
                }
                resp.getStats().inc(docToType.getOrDefault(docType, unreachableEntities));
                final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
                final Object entity = makeEntity(hitResp, hit, null);
                if (entity == null) {
                    warn("Unable to build entity " + hit.getSourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                if (entity instanceof BwShareableDbentity) {
                    final BwShareableDbentity<?> ent = (BwShareableDbentity<?>) entity;
                    principalHref = ent.getOwnerHref();
                }
                if (entity instanceof EventInfo) {
                    // This might be a single event or a recurring event.
                    final EventInfo ei = (EventInfo) entity;
                    final BwEvent ev = ei.getEvent();
                    if (ev.getRecurring()) {
                        resp.incRecurring();
                    }
                    if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                        warn("Unable to index event " + hit.getSourceAsString());
                        resp.incTotalFailed();
                        if (resp.getTotalFailed() < 50) {
                            resp.addFailure(hitResp);
                        }
                    }
                } else {
                    final EsDocInfo doc = makeDoc(resp, entity);
                    if (doc == null) {
                        if (resp.getStatus() != ok) {
                            resp.addFailure(hitResp);
                        }
                        continue;
                    }
                    final IndexRequest request = new IndexRequest(indexName);
                    request.id(doc.getId());
                    request.source(doc.getSource());
                    bulkProcessor.add(request);
                    if (entity instanceof BwEventProperty) {
                        caches.put((BwEventProperty<?>) entity);
                    }
                }
            }
            final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollResp.getScrollId());
            scrollRequest.scroll(tv);
            scrollResp = getClient().scroll(scrollRequest, RequestOptions.DEFAULT);
            // Break condition: No hits are returned
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
        try {
            bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
        } catch (final InterruptedException e) {
            errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
        }
    } catch (final Throwable t) {
        errorReturn(resp, t);
    }
    return resp;
}

Example 2 with BulkProcessor

use of org.opensearch.action.bulk.BulkProcessor in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testGlobalParametersAndSingleRequest.

public void testGlobalParametersAndSingleRequest() throws Exception {
    createIndexWithMultipleShards("test");
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    createFieldAddingPipleine("pipeline_id", "fieldNameXYZ", "valueXYZ");
    // tag::bulk-processor-mix-parameters
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setGlobalIndex("tweets").setGlobalRouting("routing").setGlobalPipeline("pipeline_id").build()) {
        processor.add(// <1>
        new IndexRequest().source(XContentType.JSON, "user", "some user"));
        processor.add(// <2>
        new IndexRequest("blogs").id("1").source(XContentType.JSON, "title", "some title"));
    }
    // end::bulk-processor-mix-parameters
    latch.await();
    Iterable<SearchHit> hits = searchAll(new SearchRequest("tweets").routing("routing"));
    assertThat(hits, everyItem(hasProperty(fieldFromSource("user"), equalTo("some user"))));
    assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
    Iterable<SearchHit> blogs = searchAll(new SearchRequest("blogs").routing("routing"));
    assertThat(blogs, everyItem(hasProperty(fieldFromSource("title"), equalTo("some title"))));
    assertThat(blogs, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
}

Example 3 with BulkProcessor

use of org.opensearch.action.bulk.BulkProcessor in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testBulkProcessorWaitOnClose.

public void testBulkProcessorWaitOnClose() throws Exception {
    BulkProcessorTestListener listener = new BulkProcessorTestListener();
    int numDocs = randomIntBetween(10, 100);
    BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(randomIntBetween(1, 10), RandomPicks.randomFrom(random(), ByteSizeUnit.values()))).build();
    MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
    assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
    if (randomBoolean()) {
        // check if we can call it multiple times
        if (randomBoolean()) {
            assertThat(processor.awaitClose(1, TimeUnit.MINUTES), is(true));
        } else {
            processor.close();
        }
    }
    assertThat(listener.beforeCounts.get(), greaterThanOrEqualTo(1));
    assertThat(listener.afterCounts.get(), greaterThanOrEqualTo(1));
    for (Throwable bulkFailure : listener.bulkFailures) {
        logger.error("bulk failure", bulkFailure);
    }
    assertThat(listener.bulkFailures.size(), equalTo(0));
    assertResponseItems(listener.bulkItems, numDocs);
    assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
}

Example 4 with BulkProcessor

use of org.opensearch.action.bulk.BulkProcessor in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testThatBulkProcessorCountIsCorrect.

public void testThatBulkProcessorCountIsCorrect() throws Exception {
    final CountDownLatch latch = new CountDownLatch(1);
    BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
    int numDocs = randomIntBetween(10, 100);
    try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
        MultiGetRequest multiGetRequest = indexDocs(processor, numDocs);
        latch.await();
        assertThat(listener.beforeCounts.get(), equalTo(1));
        assertThat(listener.afterCounts.get(), equalTo(1));
        assertThat(listener.bulkFailures.size(), equalTo(0));
        assertResponseItems(listener.bulkItems, numDocs);
        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
    }
}

Example 5 with BulkProcessor

use of org.opensearch.action.bulk.BulkProcessor in project OpenSearch by opensearch-project.

the class BulkProcessorIT method testGlobalParametersAndBulkProcessor.

public void testGlobalParametersAndBulkProcessor() throws Exception {
    createIndexWithMultipleShards("test");
    createFieldAddingPipleine("pipeline_id", "fieldNameXYZ", "valueXYZ");
    int numDocs = randomIntBetween(10, 10);
    {
        final CountDownLatch latch = new CountDownLatch(1);
        BulkProcessorTestListener listener = new BulkProcessorTestListener(latch);
        try (BulkProcessor processor = initBulkProcessorBuilder(listener).setConcurrentRequests(randomIntBetween(0, 1)).setBulkActions(numDocs).setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).setGlobalIndex("test").setGlobalRouting("routing").setGlobalPipeline("pipeline_id").build()) {
            indexDocs(processor, numDocs, null, "test", "pipeline_id");
            latch.await();
            assertThat(listener.beforeCounts.get(), equalTo(1));
            assertThat(listener.afterCounts.get(), equalTo(1));
            assertThat(listener.bulkFailures.size(), equalTo(0));
            assertResponseItems(listener.bulkItems, numDocs);
            Iterable<SearchHit> hits = searchAll(new SearchRequest("test").routing("routing"));
            assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
            assertThat(hits, containsInAnyOrder(expectedIds(numDocs)));
        }
    }
}