Examples with OneMerge org.apache.lucene.index.MergePolicy.OneMerge used on opensource projects

Example 1 with OneMerge

use of org.apache.lucene.index.MergePolicy.OneMerge in project lucene-solr by apache.

the class ConcurrentMergeScheduler method merge.

@Override
public synchronized void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
    assert !Thread.holdsLock(writer);
    initDynamicDefaults(writer);
    if (trigger == MergeTrigger.CLOSING) {
        // Disable throttling on close:
        targetMBPerSec = MAX_MERGE_MB_PER_SEC;
        updateMergeThreads();
    }
    if (verbose()) {
        message("now merge");
        message("  index: " + writer.segString());
    }
    // pending merges, until it's empty:
    while (true) {
        if (maybeStall(writer) == false) {
            break;
        }
        OneMerge merge = writer.getNextMerge();
        if (merge == null) {
            if (verbose()) {
                message("  no more merges pending; now return");
            }
            return;
        }
        boolean success = false;
        try {
            if (verbose()) {
                message("  consider merge " + writer.segString(merge.segments));
            }
            // OK to spawn a new merge thread to handle this
            // merge:
            final MergeThread newMergeThread = getMergeThread(writer, merge);
            mergeThreads.add(newMergeThread);
            updateIOThrottle(newMergeThread.merge, newMergeThread.rateLimiter);
            if (verbose()) {
                message("    launch new thread [" + newMergeThread.getName() + "]");
            }
            newMergeThread.start();
            updateMergeThreads();
            success = true;
        } finally {
            if (!success) {
                writer.mergeFinish(merge);
            }
        }
    }
}

Example 2 with OneMerge

use of org.apache.lucene.index.MergePolicy.OneMerge in project lucene-solr by apache.

the class SegmentsInfoRequestHandler method getMergeCandidatesNames.

private List<String> getMergeCandidatesNames(SolrQueryRequest req, SegmentInfos infos) throws IOException {
    List<String> result = new ArrayList<String>();
    RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
    try {
        IndexWriter indexWriter = refCounted.get();
        //get chosen merge policy
        MergePolicy mp = indexWriter.getConfig().getMergePolicy();
        //Find merges
        MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
        if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
            for (OneMerge merge : findMerges.merges) {
                //TODO: add merge grouping
                for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
                    result.add(mergeSegmentInfo.info.name);
                }
            }
        }
        return result;
    } finally {
        refCounted.decref();
    }
}

Example 3 with OneMerge

use of org.apache.lucene.index.MergePolicy.OneMerge in project lucene-solr by apache.

the class ConcurrentMergeScheduler method updateMergeThreads.

/**
   * Called whenever the running merges have changed, to set merge IO limits.
   * This method sorts the merge threads by their merge size in
   * descending order and then pauses/unpauses threads from first to last --
   * that way, smaller merges are guaranteed to run before larger ones.
   */
protected synchronized void updateMergeThreads() {
    // Only look at threads that are alive & not in the
    // process of stopping (ie have an active merge):
    final List<MergeThread> activeMerges = new ArrayList<>();
    int threadIdx = 0;
    while (threadIdx < mergeThreads.size()) {
        final MergeThread mergeThread = mergeThreads.get(threadIdx);
        if (!mergeThread.isAlive()) {
            // Prune any dead threads
            mergeThreads.remove(threadIdx);
            continue;
        }
        activeMerges.add(mergeThread);
        threadIdx++;
    }
    // Sort the merge threads, largest first:
    CollectionUtil.timSort(activeMerges);
    final int activeMergeCount = activeMerges.size();
    int bigMergeCount = 0;
    for (threadIdx = activeMergeCount - 1; threadIdx >= 0; threadIdx--) {
        MergeThread mergeThread = activeMerges.get(threadIdx);
        if (mergeThread.merge.estimatedMergeBytes > MIN_BIG_MERGE_MB * 1024 * 1024) {
            bigMergeCount = 1 + threadIdx;
            break;
        }
    }
    long now = System.nanoTime();
    StringBuilder message;
    if (verbose()) {
        message = new StringBuilder();
        message.append(String.format(Locale.ROOT, "updateMergeThreads ioThrottle=%s targetMBPerSec=%.1f MB/sec", doAutoIOThrottle, targetMBPerSec));
    } else {
        message = null;
    }
    for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++) {
        MergeThread mergeThread = activeMerges.get(threadIdx);
        OneMerge merge = mergeThread.merge;
        // pause the thread if maxThreadCount is smaller than the number of merge threads.
        final boolean doPause = threadIdx < bigMergeCount - maxThreadCount;
        double newMBPerSec;
        if (doPause) {
            newMBPerSec = 0.0;
        } else if (merge.maxNumSegments != -1) {
            newMBPerSec = forceMergeMBPerSec;
        } else if (doAutoIOThrottle == false) {
            newMBPerSec = Double.POSITIVE_INFINITY;
        } else if (merge.estimatedMergeBytes < MIN_BIG_MERGE_MB * 1024 * 1024) {
            // Don't rate limit small merges:
            newMBPerSec = Double.POSITIVE_INFINITY;
        } else {
            newMBPerSec = targetMBPerSec;
        }
        MergeRateLimiter rateLimiter = mergeThread.rateLimiter;
        double curMBPerSec = rateLimiter.getMBPerSec();
        if (verbose()) {
            long mergeStartNS = merge.mergeStartNS;
            if (mergeStartNS == -1) {
                // IndexWriter didn't start the merge yet:
                mergeStartNS = now;
            }
            message.append('\n');
            message.append(String.format(Locale.ROOT, "merge thread %s estSize=%.1f MB (written=%.1f MB) runTime=%.1fs (stopped=%.1fs, paused=%.1fs) rate=%s\n", mergeThread.getName(), bytesToMB(merge.estimatedMergeBytes), bytesToMB(rateLimiter.getTotalBytesWritten()), nsToSec(now - mergeStartNS), nsToSec(rateLimiter.getTotalStoppedNS()), nsToSec(rateLimiter.getTotalPausedNS()), rateToString(rateLimiter.getMBPerSec())));
            if (newMBPerSec != curMBPerSec) {
                if (newMBPerSec == 0.0) {
                    message.append("  now stop");
                } else if (curMBPerSec == 0.0) {
                    if (newMBPerSec == Double.POSITIVE_INFINITY) {
                        message.append("  now resume");
                    } else {
                        message.append(String.format(Locale.ROOT, "  now resume to %.1f MB/sec", newMBPerSec));
                    }
                } else {
                    message.append(String.format(Locale.ROOT, "  now change from %.1f MB/sec to %.1f MB/sec", curMBPerSec, newMBPerSec));
                }
            } else if (curMBPerSec == 0.0) {
                message.append("  leave stopped");
            } else {
                message.append(String.format(Locale.ROOT, "  leave running at %.1f MB/sec", curMBPerSec));
            }
        }
        rateLimiter.setMBPerSec(newMBPerSec);
    }
    if (verbose()) {
        message(message.toString());
    }
}