Examples with UpdateLog org.apache.solr.update.UpdateLog used on opensource projects

Example 11 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestTlogReplica method testOnlyLeaderIndexes.

public void testOnlyLeaderIndexes() throws Exception {
    createAndWaitForCollection(1, 0, 2, 0);
    CloudSolrClient cloudClient = cluster.getSolrClient();
    new UpdateRequest().add(sdoc("id", "1")).add(sdoc("id", "2")).add(sdoc("id", "3")).add(sdoc("id", "4")).process(cloudClient, collectionName);
    {
        UpdateHandler updateHandler = getSolrCore(true).get(0).getUpdateHandler();
        RefCounted<IndexWriter> iwRef = updateHandler.getSolrCoreState().getIndexWriter(null);
        assertTrue("IndexWriter at leader must see updates ", iwRef.get().hasUncommittedChanges());
        iwRef.decref();
    }
    for (SolrCore solrCore : getSolrCore(false)) {
        RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
        assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
        iwRef.decref();
    }
    checkRTG(1, 4, cluster.getJettySolrRunners());
    new UpdateRequest().deleteById("1").deleteByQuery("id:2").process(cloudClient, collectionName);
    // The DBQ is not processed at replicas, so we still can get doc2 and other docs by RTG
    checkRTG(2, 4, getSolrRunner(false));
    new UpdateRequest().commit(cloudClient, collectionName);
    waitForNumDocsInAllActiveReplicas(2);
    // Update log roll over
    for (SolrCore solrCore : getSolrCore(false)) {
        UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
        assertFalse(updateLog.hasUncommittedChanges());
    }
    // UpdateLog copy over old updates
    for (int i = 15; i <= 150; i++) {
        cloudClient.add(collectionName, sdoc("id", String.valueOf(i)));
        if (random().nextInt(100) < 15 & i != 150) {
            cloudClient.commit(collectionName);
        }
    }
    checkRTG(120, 150, cluster.getJettySolrRunners());
    waitForReplicasCatchUp(20);
}

Example 12 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecovery method testRecoveryMultipleLogs.

// in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying that all the content in the log was committed)
@Test
public void testRecoveryMultipleLogs() throws Exception {
    try {
        DirectUpdateHandler2.commitOnClose = false;
        final Semaphore logReplay = new Semaphore(0);
        final Semaphore logReplayFinish = new Semaphore(0);
        UpdateLog.testing_logReplayHook = () -> {
            try {
                assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        };
        UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
        UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
        File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
        clearIndex();
        assertU(commit());
        assertU(adoc("id", "AAAAAA"));
        assertU(adoc("id", "BBBBBB"));
        assertU(adoc("id", "CCCCCC"));
        h.close();
        String[] files = ulog.getLogList(logDir);
        Arrays.sort(files);
        String fname = files[files.length - 1];
        RandomAccessFile raf = new RandomAccessFile(new File(logDir, fname), "rw");
        // seek to end
        raf.seek(raf.length());
        raf.writeLong(0xffffffffffffffffL);
        raf.writeChars("This should be appended to a good log file, representing a bad partially written record.");
        byte[] content = new byte[(int) raf.length()];
        raf.seek(0);
        raf.readFully(content);
        raf.close();
        // Now make a newer log file with just the IDs changed.  NOTE: this may not work if log format changes too much!
        findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
        findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
        findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
        // WARNING... assumes format of .00000n where n is less than 9
        long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
        String fname2 = String.format(Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, logNumber + 1);
        raf = new RandomAccessFile(new File(logDir, fname2), "rw");
        raf.write(content);
        raf.close();
        logReplay.release(1000);
        logReplayFinish.drainPermits();
        // this is what the corrupted log currently produces... subject to change.
        ignoreException("OutOfBoundsException");
        createCore();
        assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
        resetExceptionIgnores();
        assertJQ(req("q", "*:*"), "/response/numFound==6");
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
    }
}

Example 13 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecovery method testCorruptLog.

//
// test that a corrupt tlog doesn't stop us from coming up
//
@Test
public void testCorruptLog() throws Exception {
    try {
        DirectUpdateHandler2.commitOnClose = false;
        UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
        File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
        clearIndex();
        assertU(commit());
        assertU(adoc("id", "G1"));
        assertU(adoc("id", "G2"));
        assertU(adoc("id", "G3"));
        h.close();
        String[] files = ulog.getLogList(logDir);
        Arrays.sort(files);
        RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length - 1]), "rw");
        long len = raf.length();
        // seek to start
        raf.seek(0);
        // zero out file
        raf.write(new byte[(int) len]);
        raf.close();
        // this is what the corrupted log currently produces... subject to change.
        ignoreException("Failure to open existing log file");
        createCore();
        resetExceptionIgnores();
        // just make sure it responds
        assertJQ(req("q", "*:*"), "/response/numFound==0");
        //
        // Now test that the bad log file doesn't mess up retrieving latest versions
        //
        String v104 = getNextVersion();
        String v105 = getNextVersion();
        String v106 = getNextVersion();
        updateJ(jsonAdd(sdoc("id", "G4", "_version_", v104)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "G5", "_version_", v105)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "G6", "_version_", v106)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // This currently skips the bad log file and also returns the version of the clearIndex (del *:*)
        assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[" + v106 + "," + v105 + "," + v104 + "]");
        assertU(commit());
        assertJQ(req("q", "*:*"), "/response/numFound==3");
        // This messes up some other tests (on windows) if we don't remove the bad log.
        // This *should* hopefully just be because the tests are too fragile and not because of real bugs - but it should be investigated further.
        deleteLogs();
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
    }
}

Example 14 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecovery method testDropBuffered.

@Test
public void testDropBuffered() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        String v101 = getNextVersion();
        String v102 = getNextVersion();
        String v103 = getNextVersion();
        String v104 = getNextVersion();
        String v105 = getNextVersion();
        String v200 = getNextVersion();
        String v201 = getNextVersion();
        String v203 = getNextVersion();
        String v204 = getNextVersion();
        String v205 = getNextVersion();
        String v206 = getNextVersion();
        String v301 = getNextVersion();
        String v302 = getNextVersion();
        String v998 = getNextVersion();
        String v999 = getNextVersion();
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "C1", "_version_", v101)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", v102)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C3", "_version_", v103)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C4", "_version_", v104)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C5", "_version_", v105)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        logReplay.release(1000);
        rinfoFuture = ulog.applyBufferedUpdates();
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[" + v105 + "," + v104 + "]}");
        // this time add some docs first before buffering starts (so tlog won't be at pos 0)
        updateJ(jsonAdd(sdoc("id", "C100", "_version_", v200)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C101", "_version_", v201)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C103", "_version_", v203)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C104", "_version_", v204)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C105", "_version_", v205)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C106", "_version_", v206)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        rinfoFuture = ulog.applyBufferedUpdates();
        rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'C4','_version_':" + v104 + "}" + ",{'id':'C5','_version_':" + v105 + "}" + ",{'id':'C100','_version_':" + v200 + "}" + ",{'id':'C101','_version_':" + v201 + "}" + ",{'id':'C105','_version_':" + v205 + "}" + ",{'id':'C106','_version_':" + v206 + "}" + "]");
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[" + String.join(",", v206, v205, v201, v200, v105, v104) + "]}");
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", v998)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", v999)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        // make sure we can overwrite with a lower version
        // TODO: is this functionality needed?
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", v301)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", v302)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[" + v302 + "," + v301 + "]}");
        assertJQ(req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"), "/response/docs==[" + "{'id':'C302','_version_':" + v302 + "}" + ",{'id':'C301','_version_':" + v301 + "}" + "]");
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", v302)), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}

Example 15 with UpdateLog

use of org.apache.solr.update.UpdateLog in project lucene-solr by apache.

the class TestRecoveryHdfs method testDropBuffered.

@Test
@Ignore("HDFS-3107: no truncate support yet")
public void testDropBuffered() throws Exception {
    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);
    UpdateLog.testing_logReplayHook = () -> {
        try {
            assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();
    try {
        clearIndex();
        assertU(commit());
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
        assertTrue(rinfoFuture == null);
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        // simulate updates from a leader
        updateJ(jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        logReplay.release(1000);
        rinfoFuture = ulog.applyBufferedUpdates();
        UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");
        // this time add some docs first before buffering starts (so tlog won't be at pos 0)
        updateJ(jsonAdd(sdoc("id", "C100", "_version_", "200")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C101", "_version_", "201")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C103", "_version_", "203")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C104", "_version_", "204")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        ulog.bufferUpdates();
        updateJ(jsonAdd(sdoc("id", "C105", "_version_", "205")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C106", "_version_", "206")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        rinfoFuture = ulog.applyBufferedUpdates();
        rinfo = rinfoFuture.get();
        assertEquals(2, rinfo.adds);
        assertJQ(req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'C4','_version_':104}" + ",{'id':'C5','_version_':105}" + ",{'id':'C100','_version_':200}" + ",{'id':'C101','_version_':201}" + ",{'id':'C105','_version_':205}" + ",{'id':'C106','_version_':206}" + "]");
        assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");
        ulog.bufferUpdates();
        assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", "998")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", "999")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertTrue(ulog.dropBufferedUpdates());
        // make sure we can overwrite with a lower version
        // TODO: is this functionality needed?
        updateJ(jsonAdd(sdoc("id", "C301", "_version_", "301")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        updateJ(jsonAdd(sdoc("id", "C302", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        assertU(commit());
        assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");
        assertJQ(req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"), "/response/docs==[" + "{'id':'C302','_version_':302}" + ",{'id':'C301','_version_':301}" + "]");
        updateJ(jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
        // leave each test method in a good state
        assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
    } finally {
        DirectUpdateHandler2.commitOnClose = true;
        UpdateLog.testing_logReplayHook = null;
        UpdateLog.testing_logReplayFinishHook = null;
        req().close();
    }
}