Examples with SegmentInfos org.apache.lucene.index.SegmentInfos used on opensource projects

Example 6 with SegmentInfos

use of org.apache.lucene.index.SegmentInfos in project lucene-solr by apache.

the class TestSimilarityBase method testLengthEncodingBackwardCompatibility.

public void testLengthEncodingBackwardCompatibility() throws IOException {
    Similarity similarity = RandomPicks.randomFrom(random(), sims);
    for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major }) {
        for (int length : new int[] { 1, 2, 4 }) {
            // these length values are encoded accurately on both cases
            Directory dir = newDirectory();
            // set the version on the directory
            new SegmentInfos(indexCreatedVersionMajor).commit(dir);
            IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
            Document doc = new Document();
            String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
            doc.add(new TextField("foo", value, Store.NO));
            w.addDocument(doc);
            IndexReader reader = DirectoryReader.open(w);
            IndexSearcher searcher = newSearcher(reader);
            searcher.setSimilarity(similarity);
            Term term = new Term("foo", "b");
            TermContext context = TermContext.build(reader.getContext(), term);
            SimWeight simWeight = similarity.computeWeight(1f, searcher.collectionStatistics("foo"), searcher.termStatistics(term, context));
            SimilarityBase.BasicSimScorer simScorer = (SimilarityBase.BasicSimScorer) similarity.simScorer(simWeight, reader.leaves().get(0));
            float docLength = simScorer.getLengthValue(0);
            assertEquals(length, (int) docLength);
            w.close();
            reader.close();
            dir.close();
        }
    }
}

Example 7 with SegmentInfos

use of org.apache.lucene.index.SegmentInfos in project lucene-solr by apache.

the class TestBM25Similarity method testLengthEncodingBackwardCompatibility.

public void testLengthEncodingBackwardCompatibility() throws IOException {
    Similarity similarity = new BM25Similarity();
    for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major }) {
        for (int length : new int[] { 1, 2, 4 }) {
            // these length values are encoded accurately on both cases
            Directory dir = newDirectory();
            // set the version on the directory
            new SegmentInfos(indexCreatedVersionMajor).commit(dir);
            IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
            Document doc = new Document();
            String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
            doc.add(new TextField("foo", value, Store.NO));
            w.addDocument(doc);
            IndexReader reader = DirectoryReader.open(w);
            IndexSearcher searcher = newSearcher(reader);
            searcher.setSimilarity(similarity);
            Explanation expl = searcher.explain(new TermQuery(new Term("foo", "b")), 0);
            Explanation docLen = findExplanation(expl, "fieldLength");
            assertNotNull(docLen);
            assertEquals(docLen.toString(), length, (int) docLen.getValue());
            w.close();
            reader.close();
            dir.close();
        }
    }
}

Example 8 with SegmentInfos

use of org.apache.lucene.index.SegmentInfos in project lucene-solr by apache.

the class TestClassicSimilarity method testNormEncodingBackwardCompatibility.

public void testNormEncodingBackwardCompatibility() throws IOException {
    Similarity similarity = new ClassicSimilarity();
    for (int indexCreatedVersionMajor : new int[] { Version.LUCENE_6_0_0.major, Version.LATEST.major }) {
        for (int length : new int[] { 1, 4, 16 }) {
            // these length values are encoded accurately on both cases
            Directory dir = newDirectory();
            // set the version on the directory
            new SegmentInfos(indexCreatedVersionMajor).commit(dir);
            IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(similarity));
            Document doc = new Document();
            String value = IntStream.range(0, length).mapToObj(i -> "b").collect(Collectors.joining(" "));
            doc.add(new TextField("foo", value, Store.NO));
            w.addDocument(doc);
            IndexReader reader = DirectoryReader.open(w);
            IndexSearcher searcher = newSearcher(reader);
            searcher.setSimilarity(similarity);
            Explanation expl = searcher.explain(new TermQuery(new Term("foo", "b")), 0);
            Explanation fieldNorm = findExplanation(expl, "fieldNorm");
            assertNotNull(fieldNorm);
            assertEquals(fieldNorm.toString(), 1 / Math.sqrt(length), fieldNorm.getValue(), 0f);
            w.close();
            reader.close();
            dir.close();
        }
    }
}

Example 9 with SegmentInfos

use of org.apache.lucene.index.SegmentInfos in project lucene-solr by apache.

the class SegmentsInfoRequestHandler method getSegmentsInfo.

private SimpleOrderedMap<Object> getSegmentsInfo(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    SolrIndexSearcher searcher = req.getSearcher();
    SegmentInfos infos = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory());
    List<String> mergeCandidates = getMergeCandidatesNames(req, infos);
    SimpleOrderedMap<Object> segmentInfos = new SimpleOrderedMap<>();
    SimpleOrderedMap<Object> segmentInfo = null;
    for (SegmentCommitInfo segmentCommitInfo : infos) {
        segmentInfo = getSegmentInfo(segmentCommitInfo);
        if (mergeCandidates.contains(segmentCommitInfo.info.name)) {
            segmentInfo.add("mergeCandidate", true);
        }
        segmentInfos.add((String) segmentInfo.get(NAME), segmentInfo);
    }
    return segmentInfos;
}

Example 10 with SegmentInfos

use of org.apache.lucene.index.SegmentInfos in project lucene-solr by apache.

the class ReplicaNode method finishNRTCopy.

void finishNRTCopy(CopyJob job, long startNS) throws IOException {
    CopyState copyState = job.getCopyState();
    message("top: finishNRTCopy: version=" + copyState.version + (job.getFailed() ? " FAILED" : "") + " job=" + job);
    synchronized (this) {
        if ("syncing".equals(state)) {
            state = "idle";
        }
        if (curNRTCopy == job) {
            message("top: now clear curNRTCopy; job=" + job);
            curNRTCopy = null;
        } else {
            assert job.getFailed();
            message("top: skip clear curNRTCopy: we were cancelled; job=" + job);
        }
        if (job.getFailed()) {
            return;
        }
        // Does final file renames:
        job.finish();
        // Turn byte[] back to SegmentInfos:
        byte[] infosBytes = copyState.infosBytes;
        SegmentInfos infos = SegmentInfos.readCommit(dir, new BufferedChecksumIndexInput(new ByteArrayIndexInput("SegmentInfos", copyState.infosBytes)), copyState.gen);
        assert infos.getVersion() == copyState.version;
        message("  version=" + infos.getVersion() + " segments=" + infos.toString());
        // Cutover to new searcher:
        if (mgr != null) {
            ((SegmentInfosSearcherManager) mgr).setCurrentInfos(infos);
        }
        // Must first incRef new NRT files, then decRef old ones, to make sure we don't remove an NRT file that's in common to both:
        Collection<String> newFiles = copyState.files.keySet();
        message("top: incRef newNRTFiles=" + newFiles);
        deleter.incRef(newFiles);
        // If any of our new files were previously copied merges, we clear them now, so we don't try to later delete a non-existent file:
        pendingMergeFiles.removeAll(newFiles);
        message("top: after remove from pending merges pendingMergeFiles=" + pendingMergeFiles);
        message("top: decRef lastNRTFiles=" + lastNRTFiles);
        deleter.decRef(lastNRTFiles);
        lastNRTFiles.clear();
        lastNRTFiles.addAll(newFiles);
        message("top: set lastNRTFiles=" + lastNRTFiles);
        // in an NRT point:
        if (copyState.completedMergeFiles.isEmpty() == false) {
            message("now remove-if-not-ref'd completed merge files: " + copyState.completedMergeFiles);
            for (String fileName : copyState.completedMergeFiles) {
                if (pendingMergeFiles.contains(fileName)) {
                    pendingMergeFiles.remove(fileName);
                    deleter.deleteIfNoRef(fileName);
                }
            }
        }
        lastFileMetaData = copyState.files;
    }
    int markerCount;
    IndexSearcher s = mgr.acquire();
    try {
        markerCount = s.count(new TermQuery(new Term("marker", "marker")));
    } finally {
        mgr.release(s);
    }
    message(String.format(Locale.ROOT, "top: done sync: took %.3fs for %s, opened NRT reader version=%d markerCount=%d", (System.nanoTime() - startNS) / 1000000000.0, bytesToString(job.getTotalBytesCopied()), copyState.version, markerCount));
}