Examples with IndexFingerprint org.apache.solr.update.IndexFingerprint used on opensource projects

Example 1 with IndexFingerprint

use of org.apache.solr.update.IndexFingerprint in project lucene-solr by apache.

the class SolrIndexSearcher method getIndexFingerprint.

/** @lucene.internal
   * gets a cached version of the IndexFingerprint for this searcher
   **/
public IndexFingerprint getIndexFingerprint(long maxVersion) throws IOException {
    final SolrIndexSearcher searcher = this;
    final AtomicReference<IOException> exception = new AtomicReference<>();
    try {
        return searcher.getTopReaderContext().leaves().stream().map(ctx -> {
            try {
                return searcher.getCore().getIndexFingerprint(searcher, ctx, maxVersion);
            } catch (IOException e) {
                exception.set(e);
                return null;
            }
        }).filter(java.util.Objects::nonNull).reduce(new IndexFingerprint(maxVersion), IndexFingerprint::reduce);
    } finally {
        if (exception.get() != null)
            throw exception.get();
    }
}

Example 2 with IndexFingerprint

use of org.apache.solr.update.IndexFingerprint in project lucene-solr by apache.

the class RealTimeGetComponent method processGetUpdates.

public void processGetUpdates(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    String versionsStr = params.get("getUpdates");
    if (versionsStr == null)
        return;
    UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
    if (ulog == null)
        return;
    // handle version ranges
    List<Long> versions = null;
    if (versionsStr.indexOf("...") != -1) {
        versions = resolveVersionRanges(versionsStr, ulog);
    } else {
        versions = StrUtils.splitSmart(versionsStr, ",", true).stream().map(Long::parseLong).collect(Collectors.toList());
    }
    // find fingerprint for max version for which updates are requested
    boolean doFingerprint = params.getBool("fingerprint", false);
    if (doFingerprint) {
        long maxVersionForUpdate = Collections.min(versions, PeerSync.absComparator);
        IndexFingerprint fingerprint = IndexFingerprint.getFingerprint(req.getCore(), Math.abs(maxVersionForUpdate));
        rb.rsp.add("fingerprint", fingerprint);
    }
    List<Object> updates = new ArrayList<>(versions.size());
    long minVersion = Long.MAX_VALUE;
    // TODO: get this from cache instead of rebuilding?
    try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
        for (Long version : versions) {
            try {
                Object o = recentUpdates.lookup(version);
                if (o == null)
                    continue;
                if (version > 0) {
                    minVersion = Math.min(minVersion, version);
                }
                // TODO: do any kind of validation here?
                updates.add(o);
            } catch (SolrException | ClassCastException e) {
                log.warn("Exception reading log for updates", e);
            }
        }
        // Must return all delete-by-query commands that occur after the first add requested
        // since they may apply.
        updates.addAll(recentUpdates.getDeleteByQuery(minVersion));
        rb.rsp.add("updates", updates);
    }
}

Example 3 with IndexFingerprint

use of org.apache.solr.update.IndexFingerprint in project lucene-solr by apache.

the class RealTimeGetComponent method processGetVersions.

///////////////////////////////////////////////////////////////////////////////////
// Returns last versions added to index
///////////////////////////////////////////////////////////////////////////////////
public void processGetVersions(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    int nVersions = params.getInt("getVersions", -1);
    if (nVersions == -1)
        return;
    boolean doFingerprint = params.getBool("fingerprint", false);
    String sync = params.get("sync");
    if (sync != null) {
        processSync(rb, nVersions, sync);
        return;
    }
    UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
    if (ulog == null)
        return;
    // and would avoid mismatch if documents are being actively index especially during PeerSync
    if (doFingerprint) {
        IndexFingerprint fingerprint = IndexFingerprint.getFingerprint(req.getCore(), Long.MAX_VALUE);
        rb.rsp.add("fingerprint", fingerprint);
    }
    try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
        List<Long> versions = recentUpdates.getVersions(nVersions);
        rb.rsp.add("versions", versions);
    }
}

Example 4 with IndexFingerprint

use of org.apache.solr.update.IndexFingerprint in project lucene-solr by apache.

the class SolrCore method getIndexFingerprint.

/**
   * Computes fingerprint of a segment and caches it only if all the version in segment are included in the fingerprint.
   * We can't use computeIfAbsent as caching is conditional (as described above)
   * There is chance that two threads may compute fingerprint on the same segment. It might be OK to do so rather than locking entire map.
   *
   * @param searcher   searcher that includes specified LeaderReaderContext
   * @param ctx        LeafReaderContext of a segment to compute fingerprint of
   * @param maxVersion maximum version number to consider for fingerprint computation
   * @return IndexFingerprint of the segment
   * @throws IOException Can throw IOException
   */
public IndexFingerprint getIndexFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, long maxVersion) throws IOException {
    IndexReader.CacheHelper cacheHelper = ctx.reader().getReaderCacheHelper();
    if (cacheHelper == null) {
        log.debug("Cannot cache IndexFingerprint as reader does not support caching. searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
        return IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
    }
    IndexFingerprint f = null;
    f = perSegmentFingerprintCache.get(cacheHelper.getKey());
    //
    if (f == null || (f.getMaxInHash() > maxVersion) || (f.getNumDocs() != ctx.reader().numDocs())) {
        log.debug("IndexFingerprint cache miss for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
        f = IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
        // cache fingerprint for the segment only if all the versions in the segment are included in the fingerprint
        if (f.getMaxVersionEncountered() == f.getMaxInHash()) {
            log.info("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
            perSegmentFingerprintCache.put(cacheHelper.getKey(), f);
        }
    } else {
        log.debug("IndexFingerprint cache hit for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
    log.debug("Cache Size: {}, Segments Size:{}", perSegmentFingerprintCache.size(), searcher.getTopReaderContext().leaves().size());
    return f;
}

Example 5 with IndexFingerprint

use of org.apache.solr.update.IndexFingerprint in project lucene-solr by apache.

the class RealTimeGetComponent method processGetFingeprint.

public void processGetFingeprint(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrParams params = req.getParams();
    long maxVersion = params.getLong("getFingerprint", Long.MAX_VALUE);
    IndexFingerprint fingerprint = IndexFingerprint.getFingerprint(req.getCore(), Math.abs(maxVersion));
    rb.rsp.add("fingerprint", fingerprint);
}