Examples with EsDocInfo org.bedework.util.elasticsearch.EsDocInfo used on opensource projects

Example 1 with EsDocInfo

use of org.bedework.util.elasticsearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindexEvent.

private boolean reindexEvent(final ReindexResponse.Failure resp, final String indexName, final SearchHit sh, final EventInfo ei, final BulkProcessor bulkProcessor) {
    try {
        /* If it's not recurring or a stand-alone instance index it */
        final BwEvent ev = ei.getEvent();
        if ((uidsMap != null) && (ev.getLocationUid() == null)) {
            final String locuid = uidsMap.get(ev.getUid());
            if (locuid != null) {
                uidsSet++;
                ev.setLocationUid(locuid);
            }
        }
        if (!restoreEvProps(resp, ei)) {
            return false;
        }
        if (!ev.testRecurring() && (ev.getRecurrenceId() == null)) {
            final EsDocInfo doc = makeDoc(resp, ei, ItemKind.master, ev.getDtstart(), ev.getDtend(), // ev.getRecurrenceId(),
            null, null);
            if (doc == null) {
                return false;
            }
            final IndexRequest request = new IndexRequest(indexName, sh.type(), doc.getId());
            request.source(doc.getSource());
            bulkProcessor.add(request);
            return true;
        }
        if (ev.getRecurrenceId() != null) {
            errorReturn(resp, "Not implemented - index of single override");
            return false;
        }
        if (!addOverrides(resp, idxpars.getUserIndexName(), ei)) {
            return false;
        }
        final int maxYears;
        final int maxInstances;
        final DateLimits dl = new DateLimits();
        if (ev.getPublick()) {
            maxYears = unauthpars.getMaxYears();
            maxInstances = unauthpars.getMaxInstances();
        } else {
            maxYears = authpars.getMaxYears();
            maxInstances = authpars.getMaxInstances();
        }
        final RecurPeriods rp = RecurUtil.getPeriods(ev, maxYears, maxInstances);
        if (rp.instances.isEmpty()) {
            errorReturn(resp, "No instances for an alleged recurring event.");
            return false;
        }
        final String stzid = ev.getDtstart().getTzid();
        int instanceCt = maxInstances;
        final boolean dateOnly = ev.getDtstart().getDateType();
        /* First build a table of overrides so we can skip these later
       */
        final Map<String, String> overrides = new HashMap<>();
        if (!Util.isEmpty(ei.getOverrides())) {
            for (final EventInfo oei : ei.getOverrides()) {
                final BwEvent ov = oei.getEvent();
                overrides.put(ov.getRecurrenceId(), ov.getRecurrenceId());
                final String dtstart;
                if (ov.getDtstart().getDateType()) {
                    dtstart = ov.getRecurrenceId().substring(0, 8);
                } else {
                    dtstart = ov.getRecurrenceId();
                }
                final BwDateTime rstart = BwDateTime.makeBwDateTime(ov.getDtstart().getDateType(), dtstart, stzid);
                final BwDateTime rend = rstart.addDuration(BwDuration.makeDuration(ov.getDuration()));
                final EsDocInfo doc = makeDoc(resp, oei, ItemKind.override, rstart, rend, ov.getRecurrenceId(), dl);
                if (doc == null) {
                    return false;
                }
                final IndexRequest request = new IndexRequest(indexName, sh.type(), doc.getId());
                request.source(doc.getSource());
                bulkProcessor.add(request);
                instanceCt--;
            }
        }
        for (final Period p : rp.instances) {
            String dtval = p.getStart().toString();
            if (dateOnly) {
                dtval = dtval.substring(0, 8);
            }
            final BwDateTime rstart = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
            if (overrides.get(rstart.getDate()) != null) {
                // Overrides indexed separately - skip this instance.
                continue;
            }
            final String recurrenceId = rstart.getDate();
            dtval = p.getEnd().toString();
            if (dateOnly) {
                dtval = dtval.substring(0, 8);
            }
            final BwDateTime rend = BwDateTime.makeBwDateTime(dateOnly, dtval, stzid);
            final EsDocInfo doc = makeDoc(resp, ei, entity, rstart, rend, recurrenceId, dl);
            if (doc == null) {
                return false;
            }
            final IndexRequest request = new IndexRequest(indexName, sh.type(), doc.getId());
            request.source(doc.getSource());
            bulkProcessor.add(request);
            instanceCt--;
            if (instanceCt == 0) {
                // That's all you're getting from me
                break;
            }
        }
        // </editor-fold>
        // <editor-fold desc="Emit the master event with a date range covering the entire period.">
        final BwDateTime dtstart = BwDateTime.makeBwDateTime(dateOnly, dl.minStart, stzid);
        final BwDateTime dtend = BwDateTime.makeBwDateTime(dateOnly, dl.maxEnd, stzid);
        final EsDocInfo doc = makeDoc(resp, ei, ItemKind.master, dtstart, dtend, null, null);
        if (doc == null) {
            return false;
        }
        final IndexRequest request = new IndexRequest(indexName, sh.type(), doc.getId());
        request.source(doc.getSource());
        bulkProcessor.add(request);
        // </editor-fold>
        return true;
    } catch (final Throwable t) {
        errorReturn(resp, t);
        return false;
    }
}

Example 2 with EsDocInfo

use of org.bedework.util.elasticsearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method indexEvent.

private IndexResponse indexEvent(final EventInfo ei, final ItemKind kind, final BwDateTime start, final BwDateTime end, final String recurid, final DateLimits dl) throws CalFacadeException {
    final BwEvent ev = ei.getEvent();
    try {
        final DocBuilder db = getDocBuilder();
        final EsDocInfo di = db.makeDoc(ei, kind, start, end, recurid);
        if (dl != null) {
            dl.checkMin(start);
            dl.checkMax(end);
        }
        return indexDoc(di);
    } catch (final CalFacadeException cfe) {
        throw cfe;
    } catch (final VersionConflictEngineException vcee) {
        if (vcee.getCurrentVersion() == vcee.getProvidedVersion()) {
            warn("Failed index with equal version for kind " + kind + " and href " + ev.getHref());
        }
        return null;
    } catch (final Throwable t) {
        throw new CalFacadeException(t);
    }
}

Example 3 with EsDocInfo

use of org.bedework.util.elasticsearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

private void reindex(final ReindexResponse resp, final String indexName, final String docType) {
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final Client cl = getClient(resp);
    if (cl == null) {
        return;
    }
    checkUidsMap();
    // Start with default index as source
    targetIndex = Util.buildPath(false, idxpars.getUserIndexName());
    final BulkProcessor bulkProcessor = BulkProcessor.builder(cl, new BulkListener()).setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    SearchResponse scrollResp = cl.prepareSearch(targetIndex).setSearchType(SearchType.SCAN).setScroll(tv).setQuery(qb).setSize(batchSize).execute().actionGet();
    // Switch to new index
    targetIndex = indexName;
    // Scroll until no hits are returned
    while (true) {
        for (final SearchHit hit : scrollResp.getHits().getHits()) {
            final String dtype = hit.getType();
            resp.incProcessed();
            if ((resp.getProcessed() % 250) == 0) {
                info("processed " + docType + ": " + resp.getProcessed());
            }
            if (dtype.equals(docTypeUpdateTracker)) {
                continue;
            }
            resp.getStats().inc(docToType.getOrDefault(dtype, unreachableEntities));
            final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
            final Object entity = makeEntity(hitResp, hit, null);
            if (entity == null) {
                warn("Unable to build entity " + hit.sourceAsString());
                resp.incTotalFailed();
                if (resp.getTotalFailed() < 50) {
                    resp.addFailure(hitResp);
                }
                continue;
            }
            if (entity instanceof BwShareableDbentity) {
                final BwShareableDbentity ent = (BwShareableDbentity) entity;
                try {
                    principal = BwPrincipal.makePrincipal(ent.getOwnerHref());
                } catch (final CalFacadeException cfe) {
                    errorReturn(resp, cfe);
                    return;
                }
            }
            if (entity instanceof EventInfo) {
                // This might be a single event or a recurring event.
                final EventInfo ei = (EventInfo) entity;
                final BwEvent ev = ei.getEvent();
                if (ev.getRecurring()) {
                    resp.incRecurring();
                }
                if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                    warn("Unable to iondex event " + hit.sourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                }
            } else {
                final EsDocInfo doc = makeDoc(resp, entity);
                if (doc == null) {
                    if (resp.getStatus() != ok) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                final IndexRequest request = new IndexRequest(indexName, hit.type(), doc.getId());
                request.source(doc.getSource());
                bulkProcessor.add(request);
                if (entity instanceof BwEventProperty) {
                    if (!cacheEvprop(hitResp, (BwEventProperty) entity)) {
                        resp.addFailure(hitResp);
                    }
                }
            }
        }
        scrollResp = cl.prepareSearchScroll(scrollResp.getScrollId()).setScroll(tv).execute().actionGet();
        // Break condition: No hits are returned
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
    try {
        bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
    } catch (final InterruptedException e) {
        errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
    }
    if (uidsSet > 0) {
        info("Uids set: " + uidsSet);
        info("uidOverridesSet: " + uidOverridesSet);
    }
    uidsMap = null;
    uidsOverideMap = null;
}

Example 4 with EsDocInfo

use of org.bedework.util.elasticsearch.EsDocInfo in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method makeDoc.

private EsDocInfo makeDoc(final Response resp, final EventInfo ei, final ItemKind kind, final BwDateTime start, final BwDateTime end, final String recurid, final DateLimits dl) {
    try {
        final DocBuilder db = getDocBuilder();
        final EsDocInfo di = db.makeDoc(ei, kind, start, end, recurid);
        if (dl != null) {
            dl.checkMin(start);
            dl.checkMax(end);
        }
        return di;
    } catch (final Throwable t) {
        errorReturn(resp, t);
        return null;
    }
}