Examples with BufferedChecksumIndexInput org.apache.lucene.store.BufferedChecksumIndexInput used on opensource projects

Example 11 with BufferedChecksumIndexInput

use of org.apache.lucene.store.BufferedChecksumIndexInput in project lucene-solr by apache.

the class ReplicaNode method start.

/** Start up this replica, which possibly requires heavy copying of files from the primary node, if we were down for a long time */
protected synchronized void start(long curPrimaryGen) throws IOException {
    if (state.equals("init") == false) {
        throw new IllegalStateException("already started");
    }
    message("top: now start");
    try {
        // Figure out what state our local index is in now:
        String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(dir);
        // Also look for any pending_segments_N, in case we crashed mid-commit.  We must "inflate" our infos gen to at least this, since
        // otherwise we may wind up re-using the pending_segments_N file name on commit, and then our deleter can get angry because it still
        // wants to delete this file:
        long maxPendingGen = -1;
        for (String fileName : dir.listAll()) {
            if (fileName.startsWith(IndexFileNames.PENDING_SEGMENTS)) {
                long gen = Long.parseLong(fileName.substring(IndexFileNames.PENDING_SEGMENTS.length() + 1), Character.MAX_RADIX);
                if (gen > maxPendingGen) {
                    maxPendingGen = gen;
                }
            }
        }
        SegmentInfos infos;
        if (segmentsFileName == null) {
            // No index here yet:
            infos = new SegmentInfos(Version.LATEST.major);
            message("top: init: no segments in index");
        } else {
            message("top: init: read existing segments commit " + segmentsFileName);
            infos = SegmentInfos.readCommit(dir, segmentsFileName);
            message("top: init: segments: " + infos.toString() + " version=" + infos.getVersion());
            Collection<String> indexFiles = infos.files(false);
            lastCommitFiles.add(segmentsFileName);
            lastCommitFiles.addAll(indexFiles);
            // Always protect the last commit:
            deleter.incRef(lastCommitFiles);
            lastNRTFiles.addAll(indexFiles);
            deleter.incRef(lastNRTFiles);
            message("top: commitFiles=" + lastCommitFiles);
            message("top: nrtFiles=" + lastNRTFiles);
        }
        message("top: delete unknown files on init: all files=" + Arrays.toString(dir.listAll()));
        deleter.deleteUnknownFiles(segmentsFileName);
        message("top: done delete unknown files on init: all files=" + Arrays.toString(dir.listAll()));
        String s = infos.getUserData().get(PRIMARY_GEN_KEY);
        long myPrimaryGen;
        if (s == null) {
            assert infos.size() == 0;
            myPrimaryGen = -1;
        } else {
            myPrimaryGen = Long.parseLong(s);
        }
        message("top: myPrimaryGen=" + myPrimaryGen);
        boolean doCommit;
        if (infos.size() > 0 && myPrimaryGen != -1 && myPrimaryGen != curPrimaryGen) {
            assert myPrimaryGen < curPrimaryGen;
            // Primary changed while we were down.  In this case, we must sync from primary before opening a reader, because it's possible current
            // files we have will need to be overwritten with different ones (if index rolled back and "forked"), and we can't overwrite open
            // files on Windows:
            final long initSyncStartNS = System.nanoTime();
            message("top: init: primary changed while we were down myPrimaryGen=" + myPrimaryGen + " vs curPrimaryGen=" + curPrimaryGen + "; sync now before mgr init");
            // Try until we succeed in copying over the latest NRT point:
            CopyJob job = null;
            // We may need to overwrite files referenced by our latest commit, either right now on initial sync, or on a later sync.  To make
            // sure the index is never even in an "apparently" corrupt state (where an old segments_N references invalid files) we forcefully
            // remove the commit now, and refuse to start the replica if this delete fails:
            message("top: now delete starting commit point " + segmentsFileName);
            // If this throws exc (e.g. due to virus checker), we cannot start this replica:
            assert deleter.getRefCount(segmentsFileName) == 1;
            deleter.decRef(Collections.singleton(segmentsFileName));
            if (dir instanceof FSDirectory && ((FSDirectory) dir).checkPendingDeletions()) {
                // which if we carsh, we cause corruption:
                throw new RuntimeException("replica cannot start: existing segments file=" + segmentsFileName + " must be removed in order to start, but the file delete failed");
            }
            // So we don't later try to decRef it (illegally) again:
            boolean didRemove = lastCommitFiles.remove(segmentsFileName);
            assert didRemove;
            while (true) {
                job = newCopyJob("sync on startup replica=" + name() + " myVersion=" + infos.getVersion(), null, null, true, null);
                job.start();
                message("top: init: sync sis.version=" + job.getCopyState().version);
                // NOTE: newNRTPoint detects we are still in init (mgr is null) and does not cancel our copy if a flush happens
                try {
                    job.runBlocking();
                    job.finish();
                    // Success!
                    break;
                } catch (IOException ioe) {
                    job.cancel("startup failed", ioe);
                    if (ioe.getMessage().contains("checksum mismatch after file copy")) {
                        // OK-ish
                        message("top: failed to copy: " + ioe + "; retrying");
                    } else {
                        throw ioe;
                    }
                }
            }
            lastPrimaryGen = job.getCopyState().primaryGen;
            byte[] infosBytes = job.getCopyState().infosBytes;
            SegmentInfos syncInfos = SegmentInfos.readCommit(dir, new BufferedChecksumIndexInput(new ByteArrayIndexInput("SegmentInfos", job.getCopyState().infosBytes)), job.getCopyState().gen);
            // Must always commit to a larger generation than what's currently in the index:
            syncInfos.updateGeneration(infos);
            infos = syncInfos;
            assert infos.getVersion() == job.getCopyState().version;
            message("  version=" + infos.getVersion() + " segments=" + infos.toString());
            message("top: init: incRef nrtFiles=" + job.getFileNames());
            deleter.incRef(job.getFileNames());
            message("top: init: decRef lastNRTFiles=" + lastNRTFiles);
            deleter.decRef(lastNRTFiles);
            lastNRTFiles.clear();
            lastNRTFiles.addAll(job.getFileNames());
            message("top: init: set lastNRTFiles=" + lastNRTFiles);
            lastFileMetaData = job.getCopyState().files;
            message(String.format(Locale.ROOT, "top: %d: start: done sync: took %.3fs for %s, opened NRT reader version=%d", id, (System.nanoTime() - initSyncStartNS) / 1000000000.0, bytesToString(job.getTotalBytesCopied()), job.getCopyState().version));
            doCommit = true;
        } else {
            doCommit = false;
            lastPrimaryGen = curPrimaryGen;
            message("top: same primary as before");
        }
        if (infos.getGeneration() < maxPendingGen) {
            message("top: move infos generation from " + infos.getGeneration() + " to " + maxPendingGen);
            infos.setNextWriteGeneration(maxPendingGen);
        }
        // Notify primary we started, to give it a chance to send any warming merges our way to reduce NRT latency of first sync:
        sendNewReplica();
        // Finally, we are open for business, since our index now "agrees" with the primary:
        mgr = new SegmentInfosSearcherManager(dir, this, infos, searcherFactory);
        IndexSearcher searcher = mgr.acquire();
        try {
            // TODO: this is test specific:
            int hitCount = searcher.count(new TermQuery(new Term("marker", "marker")));
            message("top: marker count=" + hitCount + " version=" + ((DirectoryReader) searcher.getIndexReader()).getVersion());
        } finally {
            mgr.release(searcher);
        }
        // Must commit after init mgr:
        if (doCommit) {
            // Very important to commit what we just sync'd over, because we removed the pre-existing commit point above if we had to
            // overwrite any files it referenced:
            commit();
        }
        message("top: done start");
        state = "idle";
    } catch (Throwable t) {
        if (t.getMessage().startsWith("replica cannot start") == false) {
            message("exc on start:");
            t.printStackTrace(printStream);
        } else {
            dir.close();
        }
        throw IOUtils.rethrowAlways(t);
    }
}

Example 12 with BufferedChecksumIndexInput

use of org.apache.lucene.store.BufferedChecksumIndexInput in project lucene-solr by apache.

the class TestCodecUtil method testCheckFooterInvalid.

public void testCheckFooterInvalid() throws Exception {
    RAMFile file = new RAMFile();
    IndexOutput output = new RAMOutputStream(file, true);
    CodecUtil.writeHeader(output, "FooBar", 5);
    output.writeString("this is the data");
    output.writeInt(CodecUtil.FOOTER_MAGIC);
    output.writeInt(0);
    // write a bogus checksum
    output.writeLong(1234567);
    output.close();
    ChecksumIndexInput input = new BufferedChecksumIndexInput(new RAMInputStream("file", file));
    CodecUtil.checkHeader(input, "FooBar", 5, 5);
    assertEquals("this is the data", input.readString());
    Exception mine = new RuntimeException("fake exception");
    RuntimeException expected = expectThrows(RuntimeException.class, () -> {
        CodecUtil.checkFooter(input, mine);
    });
    assertEquals("fake exception", expected.getMessage());
    Throwable[] suppressed = expected.getSuppressed();
    assertEquals(1, suppressed.length);
    assertTrue(suppressed[0].getMessage().contains("checksum failed"));
    input.close();
}

Example 13 with BufferedChecksumIndexInput

use of org.apache.lucene.store.BufferedChecksumIndexInput in project lucene-solr by apache.

the class TestCodecUtil method testCheckFooterValidAtFooter.

public void testCheckFooterValidAtFooter() throws Exception {
    RAMFile file = new RAMFile();
    IndexOutput output = new RAMOutputStream(file, true);
    CodecUtil.writeHeader(output, "FooBar", 5);
    output.writeString("this is the data");
    CodecUtil.writeFooter(output);
    output.close();
    ChecksumIndexInput input = new BufferedChecksumIndexInput(new RAMInputStream("file", file));
    CodecUtil.checkHeader(input, "FooBar", 5, 5);
    assertEquals("this is the data", input.readString());
    Exception mine = new RuntimeException("fake exception");
    RuntimeException expected = expectThrows(RuntimeException.class, () -> {
        CodecUtil.checkFooter(input, mine);
    });
    assertEquals("fake exception", expected.getMessage());
    Throwable[] suppressed = expected.getSuppressed();
    assertEquals(1, suppressed.length);
    assertTrue(suppressed[0].getMessage().contains("checksum passed"));
    input.close();
}