Examples with UpdateHandler org.apache.solr.update.UpdateHandler used on opensource projects

Example 1 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecovery method testBuffering.

@Test
public void testBuffering() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        Map<String, Metric> metrics = getMetrics();
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Gauge<Integer> state = (Gauge<Integer>) metrics.get("TLOG.state");
        assertEquals(UpdateLog.State.BUFFERING.ordinal(), state.getValue().intValue());
        Gauge<Integer> bufferedOps = (Gauge<Integer>) metrics.get("TLOG.buffered.ops");
        int initialOps = bufferedOps.getValue();
        Meter applyingBuffered = (Meter) metrics.get("TLOG.applyingBuffered.ops");
        long initialApplyingOps = applyingBuffered.getCount();
        String v3 = getNextVersion();
        String v940_del = "-" + getNextVersion();
        String v950_del = "-" + getNextVersion();
        String v1010 = getNextVersion();
        String v1015 = getNextVersion();
        String v1017_del = "-" + getNextVersion();
        String v1020 = getNextVersion();
        String v1030 = getNextVersion();
        String v1040 = getNextVersion();
        String v1050 = getNextVersion();
        String v1060 = getNextVersion();
        String v1070 = getNextVersion();
        String v1080 = getNextVersion();
        String v2010_del = "-" + getNextVersion();
        String v2060_del = "-" + getNextVersion();
        String v3000_del = "-" + getNextVersion();
        String versionListFirstCheck = String.join(",", v2010_del, v1030, v1020, v1017_del, v1015, v1010);
        String versionListSecondCheck = String.join(",", v3000_del, v1080, v1050, v1060, v940_del, v1040, v3, v2010_del, v1030, v1020, v1017_del, v1015, v1010);
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "B1", "_version_", v1010)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B11", "_version_", v1015)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v1017_del));
        updateJ(jsonAdd(sdoc("id", "B2", "_version_", v1020)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", v1030)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v2010_del));
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[" + versionListFirstCheck + "]}");
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[" + versionListFirstCheck + "]}");
        // updates should be buffered, so we should not see any results yet.
        assertJQ(req("q", "*:*"), "/response/numFound==0");
        // real-time get should also not show anything (this could change in the future,
        // but it's currently used for validating version numbers too, so it would
        // be bad for updates to be visible if we're just buffering.
        assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
        assertEquals(6, bufferedOps.getValue().intValue() - initialOps);
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        logReplay.release(1000);
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        assertEquals(6L, applyingBuffered.getCount() - initialApplyingOps);
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[" + versionListFirstCheck + "]}");
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // move back to recovering
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Long ver = getVer(req("qt", "/get", "id", "B3"));
        assertEquals(Long.valueOf(v1030), ver);
        // add a reordered doc that shouldn't overwrite one in the index
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", v3)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // reorder two buffered updates
        updateJ(jsonAdd(sdoc("id", "B4", "_version_", v1040)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // this update should not take affect
        deleteAndGetVersion("B4", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v940_del));
        updateJ(jsonAdd(sdoc("id", "B6", "_version_", v1060)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B5", "_version_", v1050)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B8", "_version_", v1080)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // test that delete by query is at least buffered along with everything else so it will delete the
        // currently buffered id:8 (even if it doesn't currently support versioning)
        updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v3000_del));
        assertJQ(req("qt", "/get", "getVersions", "13"), // the "3" appears because versions aren't checked while buffering
        "=={'versions':[" + versionListSecondCheck + "]}");
        logReplay.drainPermits();
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        // apply a single update
        logReplay.release(1);
        // now add another update
        updateJ(jsonAdd(sdoc("id", "B7", "_version_", v1070)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // a reordered update that should be dropped
        deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v950_del));
        deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", v2060_del));
        logReplay.release(1000);
        UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
        assertJQ(req("q", "*:*", "sort", "id asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'B3','_version_':" + v1030 + "}" + ",{'id':'B4','_version_':" + v1040 + "}" + ",{'id':'B5','_version_':" + v1050 + "}" + ",{'id':'B7','_version_':" + v1070 + "}" + "]");
        assertEquals(1, recInfo.deleteByQuery);
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        assertEquals(0, bufferedOps.getValue().intValue());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 2 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecovery method testBufferedMultipleCalls.

@Test
public void testBufferedMultipleCalls() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    Future<UpdateLog.RecoveryInfo> rinfoFuture;
    try {
        String v101 = getNextVersion();
        String v102 = getNextVersion();
        String v103 = getNextVersion();
        String v104 = getNextVersion();
        String v105 = getNextVersion();
        String v200 = getNextVersion();
        String v201 = getNextVersion();
        String v203 = getNextVersion();
        String v204 = getNextVersion();
        String v205 = getNextVersion();
        String v206 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "c1", "_version_", v101)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c2", "_version_", v102)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c3", "_version_", v103)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // call bufferUpdates again (this currently happens when recovery fails)... we should get a new starting point
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        updateJ(jsonAdd(sdoc("id", "c4", "_version_", v104)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c5", "_version_", v105)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        logReplay.release(1000);
        rinfoFuture = ulog.applyBufferedUpdates();
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[" + v105 + "," + v104 + "]}");
        // this time add some docs first before buffering starts (so tlog won't be at pos 0)
        updateJ(jsonAdd(sdoc("id", "c100", "_version_", v200)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c101", "_version_", v201)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "c103", "_version_", v203)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c104", "_version_", v204)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // call bufferUpdates again (this currently happens when recovery fails)... we should get a new starting point
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "c105", "_version_", v205)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "c106", "_version_", v206)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        rinfoFuture = ulog.applyBufferedUpdates();
        rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'c4','_version_':" + v104 + "}" + ",{'id':'c5','_version_':" + v105 + "}" + ",{'id':'c100','_version_':" + v200 + "}" + ",{'id':'c101','_version_':" + v201 + "}" + ",{'id':'c105','_version_':" + v205 + "}" + ",{'id':'c106','_version_':" + v206 + "}" + "" + "]");
        // The updates that were buffered (but never applied) still appear in recent versions!
        // This is good for some uses, but may not be good for others.
        assertJQ(req("qt", "/get", "getVersions", "11"), "=={'versions':[" + String.join(",", v206, v205, v204, v203, v201, v200, v105, v104, v103, v102, v101) + "]}");
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 3 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestTlogReplica method testOnlyLeaderIndexes.

public void testOnlyLeaderIndexes() throws Exception {
    createAndWaitForCollection(1, 0, 2, 0);
    CloudSolrClient cloudClient = cluster.getSolrClient();
    new UpdateRequest().add(sdoc("id", "1")).add(sdoc("id", "2")).add(sdoc("id", "3")).add(sdoc("id", "4")).process(cloudClient, collectionName);
    {
        UpdateHandler updateHandler = getSolrCore(true).get(0).getUpdateHandler();
        RefCounted<IndexWriter> iwRef = updateHandler.getSolrCoreState().getIndexWriter(null);
        assertTrue("IndexWriter at leader must see updates ", iwRef.get().hasUncommittedChanges());
        iwRef.decref();
    }
    for (SolrCore solrCore : getSolrCore(false)) {
        RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
        assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
        iwRef.decref();
    }
    checkRTG(1, 4, cluster.getJettySolrRunners());
    new UpdateRequest().deleteById("1").deleteByQuery("id:2").process(cloudClient, collectionName);
    // The DBQ is not processed at replicas, so we still can get doc2 and other docs by RTG
    checkRTG(2, 4, getSolrRunner(false));
    new UpdateRequest().commit(cloudClient, collectionName);
    waitForNumDocsInAllActiveReplicas(2);
    // Update log roll over
    for (SolrCore solrCore : getSolrCore(false)) {
        UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
        assertFalse(updateLog.hasUncommittedChanges());
    }
    // UpdateLog copy over old updates
    for (int i = 15; i <= 150; i++) {
        cloudClient.add(collectionName, sdoc("id", String.valueOf(i)));
        if (random().nextInt(100) < 15 & i != 150) {
            cloudClient.commit(collectionName);
        }
    }
    checkRTG(120, 150, cluster.getJettySolrRunners());
    waitForReplicasCatchUp(20);
}

Example 4 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecovery method testDropBuffered.

@Test
public void testDropBuffered() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v101 = getNextVersion();
        String v102 = getNextVersion();
        String v103 = getNextVersion();
        String v104 = getNextVersion();
        String v105 = getNextVersion();
        String v200 = getNextVersion();
        String v201 = getNextVersion();
        String v203 = getNextVersion();
        String v204 = getNextVersion();
        String v205 = getNextVersion();
        String v206 = getNextVersion();
        String v301 = getNextVersion();
        String v302 = getNextVersion();
        String v998 = getNextVersion();
        String v999 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "C1", "_version_", v101)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", v102)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C3", "_version_", v103)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C4", "_version_", v104)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C5", "_version_", v105)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        logReplay.release(1000);
        rinfoFuture = ulog.applyBufferedUpdates();
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[" + v105 + "," + v104 + "]}");
        // this time add some docs first before buffering starts (so tlog won't be at pos 0)
        updateJ(jsonAdd(sdoc("id", "C100", "_version_", v200)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C101", "_version_", v201)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C103", "_version_", v203)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C104", "_version_", v204)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C105", "_version_", v205)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C106", "_version_", v206)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        rinfoFuture = ulog.applyBufferedUpdates();
        rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'C4','_version_':" + v104 + "}" + ",{'id':'C5','_version_':" + v105 + "}" + ",{'id':'C100','_version_':" + v200 + "}" + ",{'id':'C101','_version_':" + v201 + "}" + ",{'id':'C105','_version_':" + v205 + "}" + ",{'id':'C106','_version_':" + v206 + "}" + "]");
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[" + String.join(",", v206, v205, v201, v200, v105, v104) + "]}");
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", v998)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", v999)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        // make sure we can overwrite with a lower version
        // TODO: is this functionality needed?
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", v301)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", v302)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[" + v302 + "," + v301 + "]}");
        assertJQ(req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"), "/response/docs==[" + "{'id':'C302','_version_':" + v302 + "}" + ",{'id':'C301','_version_':" + v301 + "}" + "]");
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", v302)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 5 with UpdateHandler

use of org.apache.solr.update.UpdateHandler in project lucene-solr by apache.

the class TestRecoveryHdfs method testDropBuffered.

@Test
@Ignore("HDFS-3107: no truncate support yet")
public void testDropBuffered() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        logReplay.release(1000);
        rinfoFuture = ulog.applyBufferedUpdates();
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");
        // this time add some docs first before buffering starts (so tlog won't be at pos 0)
        updateJ(jsonAdd(sdoc("id", "C100", "_version_", "200")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C101", "_version_", "201")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C103", "_version_", "203")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C104", "_version_", "204")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C105", "_version_", "205")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C106", "_version_", "206")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        rinfoFuture = ulog.applyBufferedUpdates();
        rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'C4','_version_':104}" + ",{'id':'C5','_version_':105}" + ",{'id':'C100','_version_':200}" + ",{'id':'C101','_version_':201}" + ",{'id':'C105','_version_':205}" + ",{'id':'C106','_version_':206}" + "]");
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", "998")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", "999")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        // make sure we can overwrite with a lower version
        // TODO: is this functionality needed?
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", "301")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");
        assertJQ(req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"), "/response/docs==[" + "{'id':'C302','_version_':302}" + ",{'id':'C301','_version_':301}" + "]");
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}