Examples with UpdateHandler org.apache.solr.update.UpdateHandler used on opensource projects

Example 6 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestStressRecovery method testStressRecovery.

// This version simulates updates coming from the leader and sometimes being reordered
// and tests the ability to buffer updates and apply them later
@Test
public void testStressRecovery() throws Exception {
    assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS);
    clearIndex();
    assertU(commit());
    final int commitPercent = 5 + random().nextInt(10);
    // what percent of the commits are soft
    final int softCommitPercent = 30 + random().nextInt(75);
    final int deletePercent = 4 + random().nextInt(25);
    final int deleteByQueryPercent = random().nextInt(5);
    final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
    // fewer write threads to give recovery thread more of a chance
    int nWriteThreads = 2 + random().nextInt(10);
    final int maxConcurrentCommits = nWriteThreads;
    // query variables
    final int percentRealtimeQuery = 75;
    final int percentGetLatestVersions = random().nextInt(4);
    // number of recovery loops to perform
    final AtomicLong operations = new AtomicLong(atLeast(100));
    // fewer read threads to give writers more of a chance
    int nReadThreads = 2 + random().nextInt(10);
    initModel(ndocs);
    final AtomicInteger numCommitting = new AtomicInteger();
    List<Thread> threads = new ArrayList<>();
    final AtomicLong testVersion = new AtomicLong(0);
    final UpdateHandler uHandler = h.getCore().getUpdateHandler();
    final UpdateLog uLog = uHandler.getUpdateLog();
    final VersionInfo vInfo = uLog.getVersionInfo();
    final Object stateChangeLock = new Object();
    this.visibleModel = model;
    final Semaphore[] writePermissions = new Semaphore[nWriteThreads];
    for (int i = 0; i < nWriteThreads; i++) writePermissions[i] = new Semaphore(Integer.MAX_VALUE, false);
    final Semaphore readPermission = new Semaphore(Integer.MAX_VALUE, false);
    for (int i = 0; i < nWriteThreads; i++) {
        final int threadNum = i;
        Thread thread = new Thread("WRITER" + i) {

            Random rand = new Random(random().nextInt());

            Semaphore writePermission = writePermissions[threadNum];

            @Override
            public void run() {
                try {
                    while (operations.get() > 0) {
                        writePermission.acquire();
                        int oper = rand.nextInt(10);
                        if (oper < commitPercent) {
                            if (numCommitting.incrementAndGet() <= maxConcurrentCommits) {
                                Map<Integer, DocInfo> newCommittedModel;
                                long version;
                                synchronized (globalLock) {
                                    // take a snapshot
                                    newCommittedModel = new HashMap<>(model);
                                    version = snapshotCount++;
                                }
                                synchronized (stateChangeLock) {
                                    // so change the version to -1 so we won't update our model.
                                    if (uLog.getState() != UpdateLog.State.ACTIVE)
                                        version = -1;
                                    if (rand.nextInt(100) < softCommitPercent) {
                                        verbose("softCommit start");
                                        assertU(TestHarness.commit("softCommit", "true"));
                                        verbose("softCommit end");
                                    } else {
                                        verbose("hardCommit start");
                                        assertU(commit());
                                        verbose("hardCommit end");
                                    }
                                }
                                synchronized (globalLock) {
                                    // install this model only if we are not in recovery mode.
                                    if (version >= committedModelClock) {
                                        if (VERBOSE) {
                                            verbose("installing new committedModel version=" + committedModelClock);
                                        }
                                        committedModel = newCommittedModel;
                                        committedModelClock = version;
                                    }
                                }
                            }
                            numCommitting.decrementAndGet();
                            continue;
                        }
                        int id;
                        if (rand.nextBoolean()) {
                            id = rand.nextInt(ndocs);
                        } else {
                            // reuse the last ID half of the time to force more race conditions
                            id = lastId;
                        }
                        // set the lastId before we actually change it sometimes to try and
                        // uncover more race conditions between writing and reading
                        boolean before = rand.nextBoolean();
                        if (before) {
                            lastId = id;
                        }
                        DocInfo info = model.get(id);
                        long val = info.val;
                        long nextVal = Math.abs(val) + 1;
                        // the version we set on the update should determine who wins
                        // These versions are not derived from the actual leader update handler hand hence this
                        // test may need to change depending on how we handle version numbers.
                        long version = testVersion.incrementAndGet();
                        // yield after getting the next version to increase the odds of updates happening out of order
                        if (rand.nextBoolean())
                            Thread.yield();
                        if (oper < commitPercent + deletePercent) {
                            verbose("deleting id", id, "val=", nextVal, "version", version);
                            Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
                            // but if we do return, they had better be equal
                            if (returnedVersion != null) {
                                assertEquals(-version, returnedVersion.longValue());
                            }
                            // only update model if the version is newer
                            synchronized (model) {
                                DocInfo currInfo = model.get(id);
                                if (Math.abs(version) > Math.abs(currInfo.version)) {
                                    model.put(id, new DocInfo(version, -nextVal));
                                }
                            }
                            verbose("deleting id", id, "val=", nextVal, "version", version, "DONE");
                        } else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
                            verbose("deleteByQuery id", id, "val=", nextVal, "version", version);
                            Long returnedVersion = deleteByQueryAndGetVersion("id:" + Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
                            // but if we do return, they had better be equal
                            if (returnedVersion != null) {
                                assertEquals(-version, returnedVersion.longValue());
                            }
                            // only update model if the version is newer
                            synchronized (model) {
                                DocInfo currInfo = model.get(id);
                                if (Math.abs(version) > Math.abs(currInfo.version)) {
                                    model.put(id, new DocInfo(version, -nextVal));
                                }
                            }
                            verbose("deleteByQuery id", id, "val=", nextVal, "version", version, "DONE");
                        } else {
                            verbose("adding id", id, "val=", nextVal, "version", version);
                            Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), FIELD, Long.toString(nextVal), "_version_", Long.toString(version)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
                            if (returnedVersion != null) {
                                assertEquals(version, returnedVersion.longValue());
                            }
                            // only update model if the version is newer
                            synchronized (model) {
                                DocInfo currInfo = model.get(id);
                                if (version > currInfo.version) {
                                    model.put(id, new DocInfo(version, nextVal));
                                }
                            }
                            if (VERBOSE) {
                                verbose("adding id", id, "val=", nextVal, "version", version, "DONE");
                            }
                        }
                        if (!before) {
                            lastId = id;
                        }
                    }
                } catch (Throwable e) {
                    operations.set(-1L);
                    throw new RuntimeException(e);
                }
            }
        };
        threads.add(thread);
    }
    for (int i = 0; i < nReadThreads; i++) {
        Thread thread = new Thread("READER" + i) {

            Random rand = new Random(random().nextInt());

            @Override
            public void run() {
                try {
                    while (operations.get() > 0) {
                        // throttle reads (don't completely stop)
                        readPermission.tryAcquire(10, TimeUnit.MILLISECONDS);
                        // bias toward a recently changed doc
                        int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs);
                        // when indexing, we update the index, then the model
                        // so when querying, we should first check the model, and then the index
                        boolean realTime = rand.nextInt(100) < percentRealtimeQuery;
                        DocInfo info;
                        if (realTime) {
                            info = visibleModel.get(id);
                        } else {
                            synchronized (globalLock) {
                                info = committedModel.get(id);
                            }
                        }
                        if (VERBOSE) {
                            verbose("querying id", id);
                        }
                        SolrQueryRequest sreq;
                        if (realTime) {
                            sreq = req("wt", "json", "qt", "/get", "ids", Integer.toString(id));
                        } else {
                            sreq = req("wt", "json", "q", "id:" + Integer.toString(id), "omitHeader", "true");
                        }
                        String response = h.query(sreq);
                        Map rsp = (Map) ObjectBuilder.fromJSON(response);
                        List doclist = (List) (((Map) rsp.get("response")).get("docs"));
                        if (doclist.size() == 0) {
                        // there's no info we can get back with a delete, so not much we can check without further synchronization
                        } else {
                            assertEquals(1, doclist.size());
                            long foundVal = (Long) (((Map) doclist.get(0)).get(FIELD));
                            long foundVer = (Long) (((Map) doclist.get(0)).get("_version_"));
                            if (foundVer < Math.abs(info.version) || (foundVer == info.version && foundVal != info.val)) {
                                // if the version matches, the val must
                                verbose("ERROR, id=", id, "found=", response, "model", info);
                                assertTrue(false);
                            }
                        }
                    }
                    if (rand.nextInt(100) < percentGetLatestVersions) {
                        getLatestVersions();
                    // TODO: some sort of validation that the latest version is >= to the latest version we added?
                    }
                } catch (Throwable e) {
                    operations.set(-1L);
                    throw new RuntimeException(e);
                }
            }
        };
        threads.add(thread);
    }
    for (Thread thread : threads) {
        thread.start();
    }
    int bufferedAddsApplied = 0;
    do {
        assertTrue(uLog.getState() == UpdateLog.State.ACTIVE);
        // before we start buffering updates, we want to point
        // visibleModel away from the live model.
        visibleModel = new ConcurrentHashMap<>(model);
        synchronized (stateChangeLock) {
            uLog.bufferUpdates();
        }
        assertTrue(uLog.getState() == UpdateLog.State.BUFFERING);
        // sometimes wait for a second to allow time for writers to write something
        if (random().nextBoolean())
            Thread.sleep(random().nextInt(10) + 1);
        Future<UpdateLog.RecoveryInfo> recoveryInfoF = uLog.applyBufferedUpdates();
        if (recoveryInfoF != null) {
            UpdateLog.RecoveryInfo recInfo = null;
            int writeThreadNumber = 0;
            while (recInfo == null) {
                try {
                    // wait a short period of time for recovery to complete (and to give a chance for more writers to concurrently add docs)
                    recInfo = recoveryInfoF.get(random().nextInt(100 / nWriteThreads), TimeUnit.MILLISECONDS);
                } catch (TimeoutException e) {
                    // idle one more write thread
                    verbose("Operation", operations.get(), "Draining permits for write thread", writeThreadNumber);
                    writePermissions[writeThreadNumber++].drainPermits();
                    if (writeThreadNumber >= nWriteThreads) {
                        // if we hit the end, back up and give a few write permits
                        writeThreadNumber--;
                        writePermissions[writeThreadNumber].release(random().nextInt(2) + 1);
                    }
                    // throttle readers so they don't steal too much CPU from the recovery thread
                    readPermission.drainPermits();
                }
            }
            bufferedAddsApplied += recInfo.adds;
        }
        // put all writers back at full blast
        for (Semaphore writePerm : writePermissions) {
            // I don't think semaphores check for overflow, so we need to check mow many remain
            int neededPermits = Integer.MAX_VALUE - writePerm.availablePermits();
            if (neededPermits > 0)
                writePerm.release(neededPermits);
        }
        // put back readers at full blast and point back to live model
        visibleModel = model;
        int neededPermits = Integer.MAX_VALUE - readPermission.availablePermits();
        if (neededPermits > 0)
            readPermission.release(neededPermits);
        verbose("ROUND=", operations.get());
    } while (operations.decrementAndGet() > 0);
    verbose("bufferedAddsApplied=", bufferedAddsApplied);
    for (Thread thread : threads) {
        thread.join();
    }
}

Example 7 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecoveryHdfs method testBufferingFlags.

@Test
public void testBufferingFlags() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // wait for replay to finish
        logReplayFinish.acquire();
        // since we died while buffering, we should see this last
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        //
        // Try again to ensure that the previous log replay didn't wipe out our flags
        //
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        // now do some normal non-buffered adds
        updateJ(jsonAdd(sdoc("id", "Q4", "_version_", "114")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q5", "_version_", "115")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q6", "_version_", "116")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        ulog.bufferUpdates();
        // simulate receiving no updates
        ulog.applyBufferedUpdates();
        // do another add to make sure flags are back to normal
        updateJ(jsonAdd(sdoc("id", "Q7", "_version_", "117")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // check flags on Q7
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        logReplayFinish.acquire();
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 8 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecoveryHdfs method testBuffering.

@Test
public void testBuffering() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "B1", "_version_", "1010")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B11", "_version_", "1015")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
        updateJ(jsonAdd(sdoc("id", "B2", "_version_", "1020")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", "1030")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        // updates should be buffered, so we should not see any results yet.
        assertJQ(req("q", "*:*"), "/response/numFound==0");
        // real-time get should also not show anything (this could change in the future,
        // but it's currently used for validating version numbers too, so it would
        // be bad for updates to be visible if we're just buffering.
        assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        logReplay.release(1000);
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // move back to recovering
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Long ver = getVer(req("qt", "/get", "id", "B3"));
        assertEquals(1030L, ver.longValue());
        // add a reordered doc that shouldn't overwrite one in the index
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // reorder two buffered updates
        updateJ(jsonAdd(sdoc("id", "B4", "_version_", "1040")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // this update should not take affect
        deleteAndGetVersion("B4", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-940"));
        updateJ(jsonAdd(sdoc("id", "B6", "_version_", "1060")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B5", "_version_", "1050")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B8", "_version_", "1080")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // test that delete by query is at least buffered along with everything else so it will delete the
        // currently buffered id:8 (even if it doesn't currently support versioning)
        updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));
        assertJQ(req("qt", "/get", "getVersions", "13"), // the "3" appears because versions aren't checked while buffering
        "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}");
        logReplay.drainPermits();
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        // apply a single update
        logReplay.release(1);
        // now add another update
        updateJ(jsonAdd(sdoc("id", "B7", "_version_", "1070")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // a reordered update that should be dropped
        deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));
        deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));
        logReplay.release(1000);
        UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
        assertJQ(req("q", "*:*", "sort", "id asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'B3','_version_':1030}" + ",{'id':'B4','_version_':1040}" + ",{'id':'B5','_version_':1050}" + ",{'id':'B7','_version_':1070}" + "]");
        assertEquals(1, recInfo.deleteByQuery);
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 9 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecovery method testCleanShutdown.

// make sure that log isn't needlessly replayed after a clean close
@Test
public void testCleanShutdown() throws Exception {
    DirectUpdateHandler2.commitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v1 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertU(adoc("id", "E1", "val_i", v1));
        assertU(adoc("id", "E2", "val_i", v1));
        // set to a high enough number so this test won't hang on a bug
        logReplay.release(10);
        h.close();
        createCore();
        // make sure the docs got committed
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // make sure no replay happened
        assertEquals(10, logReplay.availablePermits());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 10 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecovery method testBufferingFlags.

@Test
public void testBufferingFlags() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v101 = getNextVersion();
        String v102 = getNextVersion();
        String v103 = getNextVersion();
        String v114 = getNextVersion();
        String v115 = getNextVersion();
        String v116 = getNextVersion();
        String v117 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "Q1", "_version_", v101)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q2", "_version_", v102)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q3", "_version_", v103)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // wait for replay to finish
        logReplayFinish.acquire();
        // since we died while buffering, we should see this last
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        //
        // Try again to ensure that the previous log replay didn't wipe out our flags
        //
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        // now do some normal non-buffered adds
        updateJ(jsonAdd(sdoc("id", "Q4", "_version_", v114)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q5", "_version_", v115)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q6", "_version_", v116)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        ulog.bufferUpdates();
        // simulate receiving no updates
        ulog.applyBufferedUpdates();
        // do another add to make sure flags are back to normal
        updateJ(jsonAdd(sdoc("id", "Q7", "_version_", v117)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // check flags on Q7
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        logReplayFinish.acquire();
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}