Examples with IndexRequest org.opensearch.action.index.IndexRequest used on opensource projects

Example 1 with IndexRequest

use of org.opensearch.action.index.IndexRequest in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

@Override
public ReindexResponse reindex() {
    if (currentReindexing == null) {
        currentReindexing = new ReindexResponse(docType);
    }
    final ReindexResponse resp = currentReindexing;
    if (resp.getStatus() == processing) {
        return resp;
    }
    // Create a new index.
    final String indexName;
    try {
        indexName = newIndex();
    } catch (final Throwable t) {
        return Response.error(resp, t);
    }
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final var clResp = sch.getClient();
    if (!clResp.isOk()) {
        return Response.fromResponse(resp, clResp);
    }
    final var cl = clResp.getEntity();
    final BulkListener listener = new BulkListener();
    final BulkProcessor.Builder builder = BulkProcessor.builder((request, bulkListener) -> cl.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
    final BulkProcessor bulkProcessor = builder.setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    /*
    SearchResponse scrollResp = cl.prepareSearch(targetIndex)
                                  .setSearchType(SearchType.SCAN)
                                  .setScroll(tv)
                                  .setQuery(qb)
                                  .setSize(batchSize)
                                  .execute()
                                  .actionGet(); //100 hits per shard will be returned for each scroll
*/
    final SearchSourceBuilder ssb = new SearchSourceBuilder().size(batchSize).query(qb);
    final SearchRequest sr = new SearchRequest(targetIndex).source(ssb).scroll(tv);
    // Switch to new index
    targetIndex = indexName;
    try {
        SearchResponse scrollResp = cl.search(sr, RequestOptions.DEFAULT);
        if (scrollResp.status() != RestStatus.OK) {
            if (debug()) {
                debug("Search returned status " + scrollResp.status());
            }
        }
        // Scroll until no hits are returned
        while (true) {
            for (final SearchHit hit : scrollResp.getHits().getHits()) {
                resp.incProcessed();
                if ((resp.getProcessed() % 250) == 0) {
                    info("processed " + docType + ": " + resp.getProcessed());
                }
                resp.getStats().inc(docToType.getOrDefault(docType, unreachableEntities));
                final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
                final Object entity = makeEntity(hitResp, hit, null);
                if (entity == null) {
                    warn("Unable to build entity " + hit.getSourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                if (entity instanceof BwShareableDbentity) {
                    final BwShareableDbentity<?> ent = (BwShareableDbentity<?>) entity;
                    principalHref = ent.getOwnerHref();
                }
                if (entity instanceof EventInfo) {
                    // This might be a single event or a recurring event.
                    final EventInfo ei = (EventInfo) entity;
                    final BwEvent ev = ei.getEvent();
                    if (ev.getRecurring()) {
                        resp.incRecurring();
                    }
                    if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                        warn("Unable to index event " + hit.getSourceAsString());
                        resp.incTotalFailed();
                        if (resp.getTotalFailed() < 50) {
                            resp.addFailure(hitResp);
                        }
                    }
                } else {
                    final EsDocInfo doc = makeDoc(resp, entity);
                    if (doc == null) {
                        if (resp.getStatus() != ok) {
                            resp.addFailure(hitResp);
                        }
                        continue;
                    }
                    final IndexRequest request = new IndexRequest(indexName);
                    request.id(doc.getId());
                    request.source(doc.getSource());
                    bulkProcessor.add(request);
                    if (entity instanceof BwEventProperty) {
                        caches.put((BwEventProperty<?>) entity);
                    }
                }
            }
            final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollResp.getScrollId());
            scrollRequest.scroll(tv);
            scrollResp = getClient().scroll(scrollRequest, RequestOptions.DEFAULT);
            // Break condition: No hits are returned
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
        try {
            bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
        } catch (final InterruptedException e) {
            errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
        }
    } catch (final Throwable t) {
        errorReturn(resp, t);
    }
    return resp;
}

Example 2 with IndexRequest

use of org.opensearch.action.index.IndexRequest in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method makeRequest.

private IndexRequest makeRequest(final EventInfo ei, final ItemKind kind, final BwDateTime start, final BwDateTime end, final String recurid) throws CalFacadeException {
    final DocBuilder db = getDocBuilder();
    final EsDocInfo di = db.makeDoc(ei, kind, start, end, recurid);
    final IndexRequest req = new IndexRequest(targetIndex);
    req.id(di.getId());
    req.source(di.getSource());
    if (di.getVersion() != 0) {
        req.version(di.getVersion()).versionType(VersionType.EXTERNAL);
    }
    return req;
}

Example 3 with IndexRequest

use of org.opensearch.action.index.IndexRequest in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method indexDoc.

private IndexResponse indexDoc(final EsDocInfo di, final boolean waitForIt) throws Throwable {
    requireDocType(di.getType());
    // batchCurSize++;
    final IndexRequest req = new IndexRequest(targetIndex);
    req.id(di.getId());
    req.source(di.getSource());
    if (di.getVersion() != 0) {
        req.version(di.getVersion()).versionType(VersionType.EXTERNAL);
    }
    if (waitForIt) {
        req.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    }
    if (debug()) {
        debug("Indexing to index " + targetIndex + " with DocInfo " + di);
    }
    return getClient().index(req, RequestOptions.DEFAULT);
}

Example 4 with IndexRequest

use of org.opensearch.action.index.IndexRequest in project veilarbportefolje by navikt.

the class OpensearchTestClient method createUserInOpensearch.

public void createUserInOpensearch(OppfolgingsBruker bruker) {
    // create document
    IndexRequest indexRequest = new IndexRequest();
    indexRequest.index(indexName.getValue());
    indexRequest.id(bruker.getAktoer_id());
    indexRequest.source(toJson(bruker), XContentType.JSON);
    try {
        restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    final AktorId aktoerId = AktorId.of(bruker.getAktoer_id());
    final Optional<GetResponse> getResponse = getDocument(aktoerId);
    assertThat(getResponse).isPresent();
    assertThat(getResponse.get()).isNotNull();
}

Example 5 with IndexRequest

use of org.opensearch.action.index.IndexRequest in project veilarbportefolje by navikt.

the class OpensearchTestClient method createUserInOpensearch.

public void createUserInOpensearch(AktorId aktoerId) {
    String document = new JSONObject().put("aktoer_id", aktoerId.toString()).put("oppfolging", true).toString();
    // create document
    IndexRequest indexRequest = new IndexRequest();
    indexRequest.index(indexName.getValue());
    indexRequest.id(aktoerId.toString());
    indexRequest.source(document, XContentType.JSON);
    try {
        restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    final Optional<GetResponse> getResponse = getDocument(aktoerId);
    assertThat(getResponse).isPresent();
    assertThat(getResponse.get()).isNotNull();
}