Examples with UpdateLog org.apache.solr.update.UpdateLog used on opensource projects

Example 1 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class RealTimeGetComponent method resolveVersionRanges.

private List<Long> resolveVersionRanges(String versionsStr, UpdateLog ulog) {
    if (StringUtils.isEmpty(versionsStr)) {
        return Collections.emptyList();
    }
    List<String> ranges = StrUtils.splitSmart(versionsStr, ",", true);
    // TODO merge ranges.
    // get all the versions from updatelog and sort them
    List<Long> versionAvailable = null;
    try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
        versionAvailable = recentUpdates.getVersions(ulog.getNumRecordsToKeep());
    }
    // sort versions
    Collections.sort(versionAvailable, PeerSync.absComparator);
    // This can be done with single pass over both ranges and versionsAvailable, that would require 
    // merging ranges. We currently use Set to ensure there are no duplicates.
    Set<Long> versionsToRet = new HashSet<>(ulog.getNumRecordsToKeep());
    for (String range : ranges) {
        String[] rangeBounds = range.split("\\.{3}");
        int indexStart = Collections.binarySearch(versionAvailable, Long.valueOf(rangeBounds[1]), PeerSync.absComparator);
        int indexEnd = Collections.binarySearch(versionAvailable, Long.valueOf(rangeBounds[0]), PeerSync.absComparator);
        if (indexStart >= 0 && indexEnd >= 0) {
            // indexEnd is exclusive
            versionsToRet.addAll(versionAvailable.subList(indexStart, indexEnd + 1));
        }
    }
    // TODO do we need to sort versions using PeerSync.absComparator?
    return new ArrayList<>(versionsToRet);
}

Example 2 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class RealTimeGetComponent method processGetUpdates.

public void processGetUpdates(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    String versionsStr = params.get("getUpdates");
    if (versionsStr == null)
        return;
    UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
    if (ulog == null)
        return;
    // handle version ranges
    List<Long> versions = null;
    if (versionsStr.indexOf("...") != -1) {
        versions = resolveVersionRanges(versionsStr, ulog);
    } else {
        versions = StrUtils.splitSmart(versionsStr, ",", true).stream().map(Long::parseLong).collect(Collectors.toList());
    }
    // find fingerprint for max version for which updates are requested
    boolean doFingerprint = params.getBool("fingerprint", false);
    if (doFingerprint) {
        long maxVersionForUpdate = Collections.min(versions, PeerSync.absComparator);
        IndexFingerprint fingerprint = IndexFingerprint.getFingerprint(req.getCore(), Math.abs(maxVersionForUpdate));
        rb.rsp.add("fingerprint", fingerprint);
    }
    List<Object> updates = new ArrayList<>(versions.size());
    long minVersion = Long.MAX_VALUE;
    // TODO: get this from cache instead of rebuilding?
    try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
        for (Long version : versions) {
            try {
                Object o = recentUpdates.lookup(version);
                if (o == null)
                    continue;
                if (version > 0) {
                    minVersion = Math.min(minVersion, version);
                }
                // TODO: do any kind of validation here?
                updates.add(o);
            } catch (SolrException | ClassCastException e) {
                log.warn("Exception reading log for updates", e);
            }
        }
        // Must return all delete-by-query commands that occur after the first add requested
        // since they may apply.
        updates.addAll(recentUpdates.getDeleteByQuery(minVersion));
        rb.rsp.add("updates", updates);
    }
}

Example 3 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class RealTimeGetComponent method processGetVersions.

///////////////////////////////////////////////////////////////////////////////////
// Returns last versions added to index
///////////////////////////////////////////////////////////////////////////////////
public void processGetVersions(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    int nVersions = params.getInt("getVersions", -1);
    if (nVersions == -1)
        return;
    boolean doFingerprint = params.getBool("fingerprint", false);
    String sync = params.get("sync");
    if (sync != null) {
        processSync(rb, nVersions, sync);
        return;
    }
    UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
    if (ulog == null)
        return;
    // and would avoid mismatch if documents are being actively index especially during PeerSync
    if (doFingerprint) {
        IndexFingerprint fingerprint = IndexFingerprint.getFingerprint(req.getCore(), Long.MAX_VALUE);
        rb.rsp.add("fingerprint", fingerprint);
    }
    try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
        List<Long> versions = recentUpdates.getVersions(nVersions);
        rb.rsp.add("versions", versions);
    }
}

Example 4 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class RealTimeGetComponent method reopenRealtimeSearcherAndGet.

/**
   * Re-open the RT searcher and get the document, referred to by the idTerm, from that searcher. 
   * @return Returns the document or null if not found.
   */
private static SolrDocument reopenRealtimeSearcherAndGet(SolrCore core, Term idTerm, ReturnFields returnFields) throws IOException {
    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
    ulog.openRealtimeSearcher();
    RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
    try {
        SolrIndexSearcher searcher = searcherHolder.get();
        int docid = searcher.getFirstMatch(idTerm);
        if (docid < 0) {
            return null;
        }
        Document luceneDocument = searcher.doc(docid, returnFields.getLuceneFieldNames());
        SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
        SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
        docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(false));
        return doc;
    } finally {
        searcherHolder.decref();
    }
}

Example 5 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class IndexFetcher method moveTlogFiles.

/**
   * <p>
   *   Copy all the tlog files from the temp tlog dir to the actual tlog dir, and reset
   *   the {@link UpdateLog}. The copy will try to preserve the original tlog directory
   *   if the copy fails.
   * </p>
   * <p>
   *   This assumes that the tlog files transferred from the leader are in synch with the
   *   index files transferred from the leader. The reset of the update log relies on the version
   *   of the latest operations found in the tlog files. If the tlogs are ahead of the latest commit
   *   point, it will not copy all the needed buffered updates for the replay and it will miss
   *   some operations.
   * </p>
   */
private boolean moveTlogFiles(File tmpTlogDir) {
    UpdateLog ulog = solrCore.getUpdateHandler().getUpdateLog();
    VersionInfo vinfo = ulog.getVersionInfo();
    // block updates until the new update log is initialised
    vinfo.blockUpdates();
    try {
        // reset the update log before copying the new tlog directory
        CdcrUpdateLog.BufferedUpdates bufferedUpdates = ((CdcrUpdateLog) ulog).resetForRecovery();
        // try to move the temp tlog files to the tlog directory
        if (!copyTmpTlogFiles2Tlog(tmpTlogDir))
            return false;
        // reinitialise the update log and copy the buffered updates
        if (bufferedUpdates.tlog != null) {
            // map file path to its new backup location
            File parentDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()).getParent().toFile();
            File backupTlogDir = new File(parentDir, tmpTlogDir.getName());
            bufferedUpdates.tlog = new File(backupTlogDir, bufferedUpdates.tlog.getName());
        }
        // init the update log with the new set of tlog files, and copy the buffered updates
        ((CdcrUpdateLog) ulog).initForRecovery(bufferedUpdates.tlog, bufferedUpdates.offset);
    } catch (Exception e) {
        LOG.error("Unable to copy tlog files", e);
        return false;
    } finally {
        vinfo.unblockUpdates();
    }
    return true;
}