Examples with UpdateLog org.apache.solr.update.UpdateLog used on opensource projects

Example 26 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecoveryHdfs method testBuffering.

@Test
public void testBuffering() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "B1", "_version_", "1010")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B11", "_version_", "1015")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
        updateJ(jsonAdd(sdoc("id", "B2", "_version_", "1020")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", "1030")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        // updates should be buffered, so we should not see any results yet.
        assertJQ(req("q", "*:*"), "/response/numFound==0");
        // real-time get should also not show anything (this could change in the future,
        // but it's currently used for validating version numbers too, so it would
        // be bad for updates to be visible if we're just buffering.
        assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        logReplay.release(1000);
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // move back to recovering
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Long ver = getVer(req("qt", "/get", "id", "B3"));
        assertEquals(1030L, ver.longValue());
        // add a reordered doc that shouldn't overwrite one in the index
        updateJ(jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // reorder two buffered updates
        updateJ(jsonAdd(sdoc("id", "B4", "_version_", "1040")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // this update should not take affect
        deleteAndGetVersion("B4", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-940"));
        updateJ(jsonAdd(sdoc("id", "B6", "_version_", "1060")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B5", "_version_", "1050")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "B8", "_version_", "1080")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // test that delete by query is at least buffered along with everything else so it will delete the
        // currently buffered id:8 (even if it doesn't currently support versioning)
        updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));
        assertJQ(req("qt", "/get", "getVersions", "13"), // the "3" appears because versions aren't checked while buffering
        "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}");
        logReplay.drainPermits();
        rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        // apply a single update
        logReplay.release(1);
        // now add another update
        updateJ(jsonAdd(sdoc("id", "B7", "_version_", "1070")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // a reordered update that should be dropped
        deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));
        deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));
        logReplay.release(1000);
        UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
        assertJQ(req("q", "*:*", "sort", "id asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'B3','_version_':1030}" + ",{'id':'B4','_version_':1040}" + ",{'id':'B5','_version_':1050}" + ",{'id':'B7','_version_':1070}" + "]");
        assertEquals(1, recInfo.deleteByQuery);
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 27 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecovery method testCleanShutdown.

// make sure that log isn't needlessly replayed after a clean close
@Test
public void testCleanShutdown() throws Exception {
    DirectUpdateHandler2.commitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v1 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertU(adoc("id", "E1", "val_i", v1));
        assertU(adoc("id", "E2", "val_i", v1));
        // set to a high enough number so this test won't hang on a bug
        logReplay.release(10);
        h.close();
        createCore();
        // make sure the docs got committed
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // make sure no replay happened
        assertEquals(10, logReplay.availablePermits());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 28 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecovery method testBufferingFlags.

@Test
public void testBufferingFlags() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v101 = getNextVersion();
        String v102 = getNextVersion();
        String v103 = getNextVersion();
        String v114 = getNextVersion();
        String v115 = getNextVersion();
        String v116 = getNextVersion();
        String v117 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "Q1", "_version_", v101)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q2", "_version_", v102)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q3", "_version_", v103)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // wait for replay to finish
        logReplayFinish.acquire();
        // since we died while buffering, we should see this last
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        //
        // Try again to ensure that the previous log replay didn't wipe out our flags
        //
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
        // now do some normal non-buffered adds
        updateJ(jsonAdd(sdoc("id", "Q4", "_version_", v114)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q5", "_version_", v115)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "Q6", "_version_", v116)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        ulog.bufferUpdates();
        // simulate receiving no updates
        ulog.applyBufferedUpdates();
        // do another add to make sure flags are back to normal
        updateJ(jsonAdd(sdoc("id", "Q7", "_version_", v117)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        req.close();
        h.close();
        createCore();
        req = req();
        uhandler = req.getCore().getUpdateHandler();
        ulog = uhandler.getUpdateLog();
        // check flags on Q7
        assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
        logReplayFinish.acquire();
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 29 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecoveryHdfs method testCleanShutdown.

// make sure that log isn't needlessly replayed after a clean close
@Test
public void testCleanShutdown() throws Exception {
    DirectUpdateHandler2.commitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertU(adoc("id", "E1", "val_i", "1"));
        assertU(adoc("id", "E2", "val_i", "1"));
        // set to a high enough number so this test won't hang on a bug
        logReplay.release(10);
        h.close();
        createCore();
        // make sure the docs got committed
        assertJQ(req("q", "*:*"), "/response/numFound==2");
        // make sure no replay happened
        assertEquals(10, logReplay.availablePermits());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 30 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestSchemalessBufferedUpdates method test.

@Test
public void test() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        // Invalid date will be normalized by ParseDateField URP
        updateJ(jsonAdd(processAdd(sdoc("id", "1", "f_dt", "2017-01-04"))), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        assertJQ(req("q", "*:*"), "/response/numFound==1");
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // If the ParseDateField URP isn't ahead of the DUP, then the date won't be normalized in the buffered tlog entry,
        // and the doc won't be indexed on the replaying replica - a warning is logged as follows:
        // WARN [...] o.a.s.u.UpdateLog REYPLAY_ERR: IOException reading log
        //            org.apache.solr.common.SolrException: Invalid Date String:'2017-01-05'
        //              at org.apache.solr.util.DateMathParser.parseMath(DateMathParser.java:234)
        //              at org.apache.solr.schema.TrieField.createField(TrieField.java:725) [...]
        updateJ(jsonAdd(processAdd(sdoc("id", "2", "f_dt", "2017-01-05"))), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture != null);
        assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
        logReplay.release(1000);
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        assertU(commit());
        assertJQ(req("q", "*:*"), "/response/numFound==2");
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}