Examples with ReindexResponse org.bedework.calfacade.indexing.ReindexResponse used on opensource projects

Example 1 with ReindexResponse

use of org.bedework.calfacade.indexing.ReindexResponse in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

@Override
public ReindexResponse reindex(final String indexName) {
    final ReindexResponse resp = currentReindexing.getOrDefault(indexName, new ReindexResponse(indexName));
    if (resp.getStatus() == processing) {
        return resp;
    }
    reindex(resp, indexName, docTypeCategory);
    if (resp.getStatus() != ok) {
        return resp;
    }
    reindex(resp, indexName, docTypeContact);
    if (resp.getStatus() != ok) {
        return resp;
    }
    reindex(resp, indexName, docTypeLocation);
    if (resp.getStatus() != ok) {
        return resp;
    }
    reindex(resp, indexName, null);
    return resp;
}

Example 2 with ReindexResponse

use of org.bedework.calfacade.indexing.ReindexResponse in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

private void reindex(final ReindexResponse resp, final String indexName, final String docType) {
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final Client cl = getClient(resp);
    if (cl == null) {
        return;
    }
    checkUidsMap();
    // Start with default index as source
    targetIndex = Util.buildPath(false, idxpars.getUserIndexName());
    final BulkProcessor bulkProcessor = BulkProcessor.builder(cl, new BulkListener()).setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    SearchResponse scrollResp = cl.prepareSearch(targetIndex).setSearchType(SearchType.SCAN).setScroll(tv).setQuery(qb).setSize(batchSize).execute().actionGet();
    // Switch to new index
    targetIndex = indexName;
    // Scroll until no hits are returned
    while (true) {
        for (final SearchHit hit : scrollResp.getHits().getHits()) {
            final String dtype = hit.getType();
            resp.incProcessed();
            if ((resp.getProcessed() % 250) == 0) {
                info("processed " + docType + ": " + resp.getProcessed());
            }
            if (dtype.equals(docTypeUpdateTracker)) {
                continue;
            }
            resp.getStats().inc(docToType.getOrDefault(dtype, unreachableEntities));
            final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
            final Object entity = makeEntity(hitResp, hit, null);
            if (entity == null) {
                warn("Unable to build entity " + hit.sourceAsString());
                resp.incTotalFailed();
                if (resp.getTotalFailed() < 50) {
                    resp.addFailure(hitResp);
                }
                continue;
            }
            if (entity instanceof BwShareableDbentity) {
                final BwShareableDbentity ent = (BwShareableDbentity) entity;
                try {
                    principal = BwPrincipal.makePrincipal(ent.getOwnerHref());
                } catch (final CalFacadeException cfe) {
                    errorReturn(resp, cfe);
                    return;
                }
            }
            if (entity instanceof EventInfo) {
                // This might be a single event or a recurring event.
                final EventInfo ei = (EventInfo) entity;
                final BwEvent ev = ei.getEvent();
                if (ev.getRecurring()) {
                    resp.incRecurring();
                }
                if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                    warn("Unable to iondex event " + hit.sourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                }
            } else {
                final EsDocInfo doc = makeDoc(resp, entity);
                if (doc == null) {
                    if (resp.getStatus() != ok) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                final IndexRequest request = new IndexRequest(indexName, hit.type(), doc.getId());
                request.source(doc.getSource());
                bulkProcessor.add(request);
                if (entity instanceof BwEventProperty) {
                    if (!cacheEvprop(hitResp, (BwEventProperty) entity)) {
                        resp.addFailure(hitResp);
                    }
                }
            }
        }
        scrollResp = cl.prepareSearchScroll(scrollResp.getScrollId()).setScroll(tv).execute().actionGet();
        // Break condition: No hits are returned
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
    try {
        bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
    } catch (final InterruptedException e) {
        errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
    }
    if (uidsSet > 0) {
        info("Uids set: " + uidsSet);
        info("uidOverridesSet: " + uidOverridesSet);
    }
    uidsMap = null;
    uidsOverideMap = null;
}

Example 3 with ReindexResponse

use of org.bedework.calfacade.indexing.ReindexResponse in project bw-calendar-engine by Bedework.

the class BwIndexCtl method reindex.

@Override
public String reindex(final String indexName) {
    try {
        final ReindexResponse resp = getIndexApp().reindex(indexName);
        info(resp.toString());
        return "ok";
    } catch (final Throwable t) {
        return "Failed: " + t.getLocalizedMessage();
    }
}