Examples with ByteArrayIndexInput org.apache.lucene.store.ByteArrayIndexInput used on opensource projects

Example 1 with ByteArrayIndexInput

use of org.apache.lucene.store.ByteArrayIndexInput in project lucene-solr by apache.

the class ReplicaNode method finishNRTCopy.

void finishNRTCopy(CopyJob job, long startNS) throws IOException {
    CopyState copyState = job.getCopyState();
    message("top: finishNRTCopy: version=" + copyState.version + (job.getFailed() ? " FAILED" : "") + " job=" + job);
    synchronized (this) {
        if ("syncing".equals(state)) {
            state = "idle";
        }
        if (curNRTCopy == job) {
            message("top: now clear curNRTCopy; job=" + job);
            curNRTCopy = null;
        } else {
            assert job.getFailed();
            message("top: skip clear curNRTCopy: we were cancelled; job=" + job);
        }
        if (job.getFailed()) {
            return;
        }
        // Does final file renames:
        job.finish();
        // Turn byte[] back to SegmentInfos:
        byte[] infosBytes = copyState.infosBytes;
        SegmentInfos infos = SegmentInfos.readCommit(dir, new BufferedChecksumIndexInput(new ByteArrayIndexInput("SegmentInfos", copyState.infosBytes)), copyState.gen);
        assert infos.getVersion() == copyState.version;
        message("  version=" + infos.getVersion() + " segments=" + infos.toString());
        // Cutover to new searcher:
        if (mgr != null) {
            ((SegmentInfosSearcherManager) mgr).setCurrentInfos(infos);
        }
        // Must first incRef new NRT files, then decRef old ones, to make sure we don't remove an NRT file that's in common to both:
        Collection<String> newFiles = copyState.files.keySet();
        message("top: incRef newNRTFiles=" + newFiles);
        deleter.incRef(newFiles);
        // If any of our new files were previously copied merges, we clear them now, so we don't try to later delete a non-existent file:
        pendingMergeFiles.removeAll(newFiles);
        message("top: after remove from pending merges pendingMergeFiles=" + pendingMergeFiles);
        message("top: decRef lastNRTFiles=" + lastNRTFiles);
        deleter.decRef(lastNRTFiles);
        lastNRTFiles.clear();
        lastNRTFiles.addAll(newFiles);
        message("top: set lastNRTFiles=" + lastNRTFiles);
        // in an NRT point:
        if (copyState.completedMergeFiles.isEmpty() == false) {
            message("now remove-if-not-ref'd completed merge files: " + copyState.completedMergeFiles);
            for (String fileName : copyState.completedMergeFiles) {
                if (pendingMergeFiles.contains(fileName)) {
                    pendingMergeFiles.remove(fileName);
                    deleter.deleteIfNoRef(fileName);
                }
            }
        }
        lastFileMetaData = copyState.files;
    }
    int markerCount;
    IndexSearcher s = mgr.acquire();
    try {
        markerCount = s.count(new TermQuery(new Term("marker", "marker")));
    } finally {
        mgr.release(s);
    }
    message(String.format(Locale.ROOT, "top: done sync: took %.3fs for %s, opened NRT reader version=%d markerCount=%d", (System.nanoTime() - startNS) / 1000000000.0, bytesToString(job.getTotalBytesCopied()), copyState.version, markerCount));
}

Example 2 with ByteArrayIndexInput

use of org.apache.lucene.store.ByteArrayIndexInput in project lucene-solr by apache.

the class ReplicaNode method start.

/** Start up this replica, which possibly requires heavy copying of files from the primary node, if we were down for a long time */
protected synchronized void start(long curPrimaryGen) throws IOException {
    if (state.equals("init") == false) {
        throw new IllegalStateException("already started");
    }
    message("top: now start");
    try {
        // Figure out what state our local index is in now:
        String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(dir);
        // Also look for any pending_segments_N, in case we crashed mid-commit.  We must "inflate" our infos gen to at least this, since
        // otherwise we may wind up re-using the pending_segments_N file name on commit, and then our deleter can get angry because it still
        // wants to delete this file:
        long maxPendingGen = -1;
        for (String fileName : dir.listAll()) {
            if (fileName.startsWith(IndexFileNames.PENDING_SEGMENTS)) {
                long gen = Long.parseLong(fileName.substring(IndexFileNames.PENDING_SEGMENTS.length() + 1), Character.MAX_RADIX);
                if (gen > maxPendingGen) {
                    maxPendingGen = gen;
                }
            }
        }
        SegmentInfos infos;
        if (segmentsFileName == null) {
            // No index here yet:
            infos = new SegmentInfos(Version.LATEST.major);
            message("top: init: no segments in index");
        } else {
            message("top: init: read existing segments commit " + segmentsFileName);
            infos = SegmentInfos.readCommit(dir, segmentsFileName);
            message("top: init: segments: " + infos.toString() + " version=" + infos.getVersion());
            Collection<String> indexFiles = infos.files(false);
            lastCommitFiles.add(segmentsFileName);
            lastCommitFiles.addAll(indexFiles);
            // Always protect the last commit:
            deleter.incRef(lastCommitFiles);
            lastNRTFiles.addAll(indexFiles);
            deleter.incRef(lastNRTFiles);
            message("top: commitFiles=" + lastCommitFiles);
            message("top: nrtFiles=" + lastNRTFiles);
        }
        message("top: delete unknown files on init: all files=" + Arrays.toString(dir.listAll()));
        deleter.deleteUnknownFiles(segmentsFileName);
        message("top: done delete unknown files on init: all files=" + Arrays.toString(dir.listAll()));
        String s = infos.getUserData().get(PRIMARY_GEN_KEY);
        long myPrimaryGen;
        if (s == null) {
            assert infos.size() == 0;
            myPrimaryGen = -1;
        } else {
            myPrimaryGen = Long.parseLong(s);
        }
        message("top: myPrimaryGen=" + myPrimaryGen);
        boolean doCommit;
        if (infos.size() > 0 && myPrimaryGen != -1 && myPrimaryGen != curPrimaryGen) {
            assert myPrimaryGen < curPrimaryGen;
            // Primary changed while we were down.  In this case, we must sync from primary before opening a reader, because it's possible current
            // files we have will need to be overwritten with different ones (if index rolled back and "forked"), and we can't overwrite open
            // files on Windows:
            final long initSyncStartNS = System.nanoTime();
            message("top: init: primary changed while we were down myPrimaryGen=" + myPrimaryGen + " vs curPrimaryGen=" + curPrimaryGen + "; sync now before mgr init");
            // Try until we succeed in copying over the latest NRT point:
            CopyJob job = null;
            // We may need to overwrite files referenced by our latest commit, either right now on initial sync, or on a later sync.  To make
            // sure the index is never even in an "apparently" corrupt state (where an old segments_N references invalid files) we forcefully
            // remove the commit now, and refuse to start the replica if this delete fails:
            message("top: now delete starting commit point " + segmentsFileName);
            // If this throws exc (e.g. due to virus checker), we cannot start this replica:
            assert deleter.getRefCount(segmentsFileName) == 1;
            deleter.decRef(Collections.singleton(segmentsFileName));
            if (dir instanceof FSDirectory && ((FSDirectory) dir).checkPendingDeletions()) {
                // which if we carsh, we cause corruption:
                throw new RuntimeException("replica cannot start: existing segments file=" + segmentsFileName + " must be removed in order to start, but the file delete failed");
            }
            // So we don't later try to decRef it (illegally) again:
            boolean didRemove = lastCommitFiles.remove(segmentsFileName);
            assert didRemove;
            while (true) {
                job = newCopyJob("sync on startup replica=" + name() + " myVersion=" + infos.getVersion(), null, null, true, null);
                job.start();
                message("top: init: sync sis.version=" + job.getCopyState().version);
                // NOTE: newNRTPoint detects we are still in init (mgr is null) and does not cancel our copy if a flush happens
                try {
                    job.runBlocking();
                    job.finish();
                    // Success!
                    break;
                } catch (IOException ioe) {
                    job.cancel("startup failed", ioe);
                    if (ioe.getMessage().contains("checksum mismatch after file copy")) {
                        // OK-ish
                        message("top: failed to copy: " + ioe + "; retrying");
                    } else {
                        throw ioe;
                    }
                }
            }
            lastPrimaryGen = job.getCopyState().primaryGen;
            byte[] infosBytes = job.getCopyState().infosBytes;
            SegmentInfos syncInfos = SegmentInfos.readCommit(dir, new BufferedChecksumIndexInput(new ByteArrayIndexInput("SegmentInfos", job.getCopyState().infosBytes)), job.getCopyState().gen);
            // Must always commit to a larger generation than what's currently in the index:
            syncInfos.updateGeneration(infos);
            infos = syncInfos;
            assert infos.getVersion() == job.getCopyState().version;
            message("  version=" + infos.getVersion() + " segments=" + infos.toString());
            message("top: init: incRef nrtFiles=" + job.getFileNames());
            deleter.incRef(job.getFileNames());
            message("top: init: decRef lastNRTFiles=" + lastNRTFiles);
            deleter.decRef(lastNRTFiles);
            lastNRTFiles.clear();
            lastNRTFiles.addAll(job.getFileNames());
            message("top: init: set lastNRTFiles=" + lastNRTFiles);
            lastFileMetaData = job.getCopyState().files;
            message(String.format(Locale.ROOT, "top: %d: start: done sync: took %.3fs for %s, opened NRT reader version=%d", id, (System.nanoTime() - initSyncStartNS) / 1000000000.0, bytesToString(job.getTotalBytesCopied()), job.getCopyState().version));
            doCommit = true;
        } else {
            doCommit = false;
            lastPrimaryGen = curPrimaryGen;
            message("top: same primary as before");
        }
        if (infos.getGeneration() < maxPendingGen) {
            message("top: move infos generation from " + infos.getGeneration() + " to " + maxPendingGen);
            infos.setNextWriteGeneration(maxPendingGen);
        }
        // Notify primary we started, to give it a chance to send any warming merges our way to reduce NRT latency of first sync:
        sendNewReplica();
        // Finally, we are open for business, since our index now "agrees" with the primary:
        mgr = new SegmentInfosSearcherManager(dir, this, infos, searcherFactory);
        IndexSearcher searcher = mgr.acquire();
        try {
            // TODO: this is test specific:
            int hitCount = searcher.count(new TermQuery(new Term("marker", "marker")));
            message("top: marker count=" + hitCount + " version=" + ((DirectoryReader) searcher.getIndexReader()).getVersion());
        } finally {
            mgr.release(searcher);
        }
        // Must commit after init mgr:
        if (doCommit) {
            // Very important to commit what we just sync'd over, because we removed the pre-existing commit point above if we had to
            // overwrite any files it referenced:
            commit();
        }
        message("top: done start");
        state = "idle";
    } catch (Throwable t) {
        if (t.getMessage().startsWith("replica cannot start") == false) {
            message("exc on start:");
            t.printStackTrace(printStream);
        } else {
            dir.close();
        }
        throw IOUtils.rethrowAlways(t);
    }
}