Examples with EsDocInfo org.bedework.util.opensearch.EsDocInfo used on opensource projects

Example 1 with EsDocInfo

use of org.bedework.util.opensearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method makeDoc.

private EsDocInfo makeDoc(final Response resp, final EventInfo ei, final ItemKind kind, final BwDateTime start, final BwDateTime end, final String recurid, final DateLimits dl) {
    try {
        final DocBuilder db = getDocBuilder();
        final EsDocInfo di = db.makeDoc(ei, kind, start, end, recurid);
        if (dl != null) {
            dl.checkMin(start);
            dl.checkMax(end);
        }
        return di;
    } catch (final Throwable t) {
        errorReturn(resp, t);
        return null;
    }
}

Example 2 with EsDocInfo

use of org.bedework.util.opensearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

@Override
public ReindexResponse reindex() {
    if (currentReindexing == null) {
        currentReindexing = new ReindexResponse(docType);
    }
    final ReindexResponse resp = currentReindexing;
    if (resp.getStatus() == processing) {
        return resp;
    }
    // Create a new index.
    final String indexName;
    try {
        indexName = newIndex();
    } catch (final Throwable t) {
        return Response.error(resp, t);
    }
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final var clResp = sch.getClient();
    if (!clResp.isOk()) {
        return Response.fromResponse(resp, clResp);
    }
    final var cl = clResp.getEntity();
    final BulkListener listener = new BulkListener();
    final BulkProcessor.Builder builder = BulkProcessor.builder((request, bulkListener) -> cl.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener);
    final BulkProcessor bulkProcessor = builder.setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    /*
    SearchResponse scrollResp = cl.prepareSearch(targetIndex)
                                  .setSearchType(SearchType.SCAN)
                                  .setScroll(tv)
                                  .setQuery(qb)
                                  .setSize(batchSize)
                                  .execute()
                                  .actionGet(); //100 hits per shard will be returned for each scroll
*/
    final SearchSourceBuilder ssb = new SearchSourceBuilder().size(batchSize).query(qb);
    final SearchRequest sr = new SearchRequest(targetIndex).source(ssb).scroll(tv);
    // Switch to new index
    targetIndex = indexName;
    try {
        SearchResponse scrollResp = cl.search(sr, RequestOptions.DEFAULT);
        if (scrollResp.status() != RestStatus.OK) {
            if (debug()) {
                debug("Search returned status " + scrollResp.status());
            }
        }
        // Scroll until no hits are returned
        while (true) {
            for (final SearchHit hit : scrollResp.getHits().getHits()) {
                resp.incProcessed();
                if ((resp.getProcessed() % 250) == 0) {
                    info("processed " + docType + ": " + resp.getProcessed());
                }
                resp.getStats().inc(docToType.getOrDefault(docType, unreachableEntities));
                final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
                final Object entity = makeEntity(hitResp, hit, null);
                if (entity == null) {
                    warn("Unable to build entity " + hit.getSourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                if (entity instanceof BwShareableDbentity) {
                    final BwShareableDbentity<?> ent = (BwShareableDbentity<?>) entity;
                    principalHref = ent.getOwnerHref();
                }
                if (entity instanceof EventInfo) {
                    // This might be a single event or a recurring event.
                    final EventInfo ei = (EventInfo) entity;
                    final BwEvent ev = ei.getEvent();
                    if (ev.getRecurring()) {
                        resp.incRecurring();
                    }
                    if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                        warn("Unable to index event " + hit.getSourceAsString());
                        resp.incTotalFailed();
                        if (resp.getTotalFailed() < 50) {
                            resp.addFailure(hitResp);
                        }
                    }
                } else {
                    final EsDocInfo doc = makeDoc(resp, entity);
                    if (doc == null) {
                        if (resp.getStatus() != ok) {
                            resp.addFailure(hitResp);
                        }
                        continue;
                    }
                    final IndexRequest request = new IndexRequest(indexName);
                    request.id(doc.getId());
                    request.source(doc.getSource());
                    bulkProcessor.add(request);
                    if (entity instanceof BwEventProperty) {
                        caches.put((BwEventProperty<?>) entity);
                    }
                }
            }
            final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollResp.getScrollId());
            scrollRequest.scroll(tv);
            scrollResp = getClient().scroll(scrollRequest, RequestOptions.DEFAULT);
            // Break condition: No hits are returned
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
        try {
            bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
        } catch (final InterruptedException e) {
            errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
        }
    } catch (final Throwable t) {
        errorReturn(resp, t);
    }
    return resp;
}

Example 3 with EsDocInfo

use of org.bedework.util.opensearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method makeRequest.

private IndexRequest makeRequest(final EventInfo ei, final ItemKind kind, final BwDateTime start, final BwDateTime end, final String recurid) throws CalFacadeException {
    final DocBuilder db = getDocBuilder();
    final EsDocInfo di = db.makeDoc(ei, kind, start, end, recurid);
    final IndexRequest req = new IndexRequest(targetIndex);
    req.id(di.getId());
    req.source(di.getSource());
    if (di.getVersion() != 0) {
        req.version(di.getVersion()).versionType(VersionType.EXTERNAL);
    }
    return req;
}

Example 4 with EsDocInfo

use of org.bedework.util.opensearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindexEvent.

private boolean reindexEvent(final ReindexResponse.Failure resp, final String indexName, final SearchHit sh, final EventInfo ei, final BulkProcessor bulkProcessor) {
    try {
        /* If it's not recurring or a stand-alone instance index it */
        final BwEvent ev = ei.getEvent();
        if (!ev.isRecurringEntity() && (ev.getRecurrenceId() == null)) {
            final EsDocInfo doc = makeDoc(resp, ei, ItemKind.master, ev.getDtstart(), ev.getDtend(), // ev.getRecurrenceId(),
            null, null);
            if (doc == null) {
                return false;
            }
            final IndexRequest request = new IndexRequest(indexName);
            request.id(doc.getId());
            request.source(doc.getSource());
            bulkProcessor.add(request);
            return true;
        }
        if (ev.getRecurrenceId() != null) {
            errorReturn(resp, "Not implemented - index of single override");
            return false;
        }
        if (!addOverrides(resp, ei)) {
            return false;
        }
        final int maxYears;
        final int maxInstances;
        final DateLimits dl = new DateLimits();
        if (ev.getPublick()) {
            maxYears = unauthpars.getMaxYears();
            maxInstances = unauthpars.getMaxInstances();
        } else {
            maxYears = authpars.getMaxYears();
            maxInstances = authpars.getMaxInstances();
        }
        final RecurPeriods rp = RecurUtil.getPeriods(ev, maxYears, maxInstances);
        if (rp.instances.isEmpty()) {
            errorReturn(resp, "No instances for an alleged recurring event.");
            return false;
        }
        final String stzid = ev.getDtstart().getTzid();
        int instanceCt = maxInstances;
        final boolean dateOnly = ev.getDtstart().getDateType();
        /* First build a table of overrides so we can skip these later
       */
        final Map<String, String> overrides = new HashMap<>();
        if (!Util.isEmpty(ei.getOverrides())) {
            for (final EventInfo oei : ei.getOverrides()) {
                final BwEvent ov = oei.getEvent();
                overrides.put(ov.getRecurrenceId(), ov.getRecurrenceId());
                final String dtstart;
                if (ov.getDtstart().getDateType()) {
                    dtstart = ov.getRecurrenceId().substring(0, 8);
                } else {
                    dtstart = ov.getRecurrenceId();
                }
                final BwDateTime rstart = BwDateTime.makeBwDateTime(ov.getDtstart().getDateType(), dtstart, stzid);
                final BwDateTime rend = rstart.addDuration(BwDuration.makeDuration(ov.getDuration()));
                final EsDocInfo doc = makeDoc(resp, oei, ItemKind.override, rstart, rend, ov.getRecurrenceId(), dl);
                if (doc == null) {
                    return false;
                }
                final IndexRequest request = new IndexRequest(indexName);
                request.id(doc.getId());
                request.source(doc.getSource());
                bulkProcessor.add(request);
                instanceCt--;
            }
        }
        for (final Period p : rp.instances) {
            String dtval = p.getStart().toString();
            if (dateOnly) {
                dtval = dtval.substring(0, 8);
            }
            final BwDateTime rstart = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
            if (overrides.get(rstart.getDate()) != null) {
                // Overrides indexed separately - skip this instance.
                continue;
            }
            final String recurrenceId = rstart.getDate();
            dtval = p.getEnd().toString();
            if (dateOnly) {
                dtval = dtval.substring(0, 8);
            }
            final BwDateTime rend = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
            final EsDocInfo doc = makeDoc(resp, ei, entity, rstart, rend, recurrenceId, dl);
            if (doc == null) {
                return false;
            }
            final IndexRequest request = new IndexRequest(indexName);
            request.id(doc.getId());
            request.source(doc.getSource());
            bulkProcessor.add(request);
            instanceCt--;
            if (instanceCt == 0) {
                // That's all you're getting from me
                break;
            }
        }
        // </editor-fold>
        // <editor-fold desc="Emit the master event with a date range covering the entire period.">
        final BwDateTime dtstart = BwDateTime.makeBwDateTime(dateOnly, dl.minStart, stzid);
        final BwDateTime dtend = BwDateTime.makeBwDateTime(dateOnly, dl.maxEnd, stzid);
        final EsDocInfo doc = makeDoc(resp, ei, ItemKind.master, dtstart, dtend, null, null);
        if (doc == null) {
            return false;
        }
        final IndexRequest request = new IndexRequest(indexName);
        request.id(doc.getId());
        request.source(doc.getSource());
        bulkProcessor.add(request);
        // </editor-fold>
        return true;
    } catch (final Throwable t) {
        errorReturn(resp, t);
        return false;
    }
}

Example 5 with EsDocInfo

use of org.bedework.util.opensearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method index.

/* Return the response after indexing */
private IndexResponse index(final Object rec, final boolean waitForIt, final boolean forTouch) throws CalFacadeException {
    EsDocInfo di = null;
    try {
        if (rec instanceof EventInfo) {
            mustBe(docTypeEvent);
            return indexEvent((EventInfo) rec, waitForIt);
        }
        final DocBuilder db = getDocBuilder();
        if (rec instanceof BwCalendar) {
            mustBe(docTypeCollection);
            di = db.makeDoc((BwCalendar) rec);
        }
        if (rec instanceof BwCategory) {
            mustBe(docTypeCategory);
            di = db.makeDoc((BwCategory) rec);
        }
        if (rec instanceof BwContact) {
            mustBe(docTypeContact);
            di = db.makeDoc((BwContact) rec);
        }
        if (rec instanceof BwLocation) {
            mustBe(docTypeLocation);
            di = db.makeDoc((BwLocation) rec);
        }
        if (rec instanceof BwPrincipal) {
            mustBe(docTypePrincipal);
            di = db.makeDoc((BwPrincipal) rec);
        }
        if (rec instanceof BwPreferences) {
            mustBe(docTypePreferences);
            di = db.makeDoc((BwPreferences) rec);
        }
        if (rec instanceof BwResource) {
            mustBe(docTypeResource);
            di = db.makeDoc((BwResource) rec);
        }
        if (rec instanceof BwResourceContent) {
            mustBe(docTypeResourceContent);
            di = db.makeDoc((BwResourceContent) rec);
        }
        if (rec instanceof BwFilterDef) {
            mustBe(docTypeFilter);
            di = db.makeDoc((BwFilterDef) rec);
        }
        if (di != null) {
            return indexDoc(di, waitForIt);
        }
        throw new CalFacadeException(new IndexException(IndexException.unknownRecordType, rec.getClass().getName()));
    } catch (final CalFacadeException cfe) {
        throw cfe;
    } catch (final VersionConflictEngineException vcee) {
        if (forTouch) {
            // Ignore - already touched
            return null;
        }
        error(vcee);
        throw new CalFacadeException(vcee);
    /* Can't do this any more
      if (vcee.currentVersion() == vcee.getProvidedVersion()) {
        warn("Failed index with equal version for type " + 
                     di.getType() +
                     " and id " + di.getId());
      }

      return null;
      */
    } catch (final Throwable t) {
        throw new CalFacadeException(t);
    }
}