Examples with MockAnalyzer org.apache.lucene.analysis.MockAnalyzer used on opensource projects

Example 16 with MockAnalyzer

use of org.apache.lucene.analysis.MockAnalyzer in project lucene-solr by apache.

the class TestBackwardsCompatibility method testUnsupportedOldIndexes.

/** This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate on too old indexes! */
public void testUnsupportedOldIndexes() throws Exception {
    for (int i = 0; i < unsupportedNames.length; i++) {
        if (VERBOSE) {
            System.out.println("TEST: index " + unsupportedNames[i]);
        }
        Path oldIndexDir = createTempDir(unsupportedNames[i]);
        TestUtil.unzip(getDataInputStream("unsupported." + unsupportedNames[i] + ".zip"), oldIndexDir);
        BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir);
        // don't checkindex, these are intentionally not supported
        dir.setCheckIndexOnClose(false);
        IndexReader reader = null;
        IndexWriter writer = null;
        try {
            reader = DirectoryReader.open(dir);
            fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);
        } catch (IndexFormatTooOldException e) {
            if (e.getReason() != null) {
                assertNull(e.getVersion());
                assertNull(e.getMinVersion());
                assertNull(e.getMaxVersion());
                assertEquals(e.getMessage(), new IndexFormatTooOldException(e.getResourceDescription(), e.getReason()).getMessage());
            } else {
                assertNotNull(e.getVersion());
                assertNotNull(e.getMinVersion());
                assertNotNull(e.getMaxVersion());
                assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());
                assertTrue(e.getMessage(), e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());
                assertEquals(e.getMessage(), new IndexFormatTooOldException(e.getResourceDescription(), e.getVersion(), e.getMinVersion(), e.getMaxVersion()).getMessage());
            }
            // pass
            if (VERBOSE) {
                System.out.println("TEST: got expected exc:");
                e.printStackTrace(System.out);
            }
        } finally {
            if (reader != null)
                reader.close();
            reader = null;
        }
        try {
            writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setCommitOnClose(false));
            fail("IndexWriter creation should not pass for " + unsupportedNames[i]);
        } catch (IndexFormatTooOldException e) {
            if (e.getReason() != null) {
                assertNull(e.getVersion());
                assertNull(e.getMinVersion());
                assertNull(e.getMaxVersion());
                assertEquals(e.getMessage(), new IndexFormatTooOldException(e.getResourceDescription(), e.getReason()).getMessage());
            } else {
                assertNotNull(e.getVersion());
                assertNotNull(e.getMinVersion());
                assertNotNull(e.getMaxVersion());
                assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());
                assertTrue(e.getMessage(), e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());
                assertEquals(e.getMessage(), new IndexFormatTooOldException(e.getResourceDescription(), e.getVersion(), e.getMinVersion(), e.getMaxVersion()).getMessage());
            }
            // pass
            if (VERBOSE) {
                System.out.println("TEST: got expected exc:");
                e.printStackTrace(System.out);
            }
            // Make sure exc message includes a path=
            assertTrue("got exc message: " + e.getMessage(), e.getMessage().indexOf("path=\"") != -1);
        } finally {
            // above, so close without waiting for merges.
            if (writer != null) {
                try {
                    writer.commit();
                } finally {
                    writer.close();
                }
            }
            writer = null;
        }
        ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
        CheckIndex checker = new CheckIndex(dir);
        checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
        CheckIndex.Status indexStatus = checker.checkIndex();
        assertFalse(indexStatus.clean);
        assertTrue(bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName()));
        checker.close();
        dir.close();
    }
}

Example 17 with MockAnalyzer

use of org.apache.lucene.analysis.MockAnalyzer in project lucene-solr by apache.

the class TestBackwardsCompatibility method testUpgradeOldSingleSegmentIndexWithAdditions.

public void testUpgradeOldSingleSegmentIndexWithAdditions() throws Exception {
    for (String name : oldSingleSegmentNames) {
        if (VERBOSE) {
            System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" + name);
        }
        Directory dir = newDirectory(oldIndexDirs.get(name));
        assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
        int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
        // create a bunch of dummy segments
        int id = 40;
        RAMDirectory ramDir = new RAMDirectory();
        for (int i = 0; i < 3; i++) {
            // only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
            MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
            IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(mp);
            IndexWriter w = new IndexWriter(ramDir, iwc);
            // add few more docs:
            for (int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {
                addDoc(w, id++);
            }
            try {
                w.commit();
            } finally {
                w.close();
            }
        }
        // add dummy segments (which are all in current
        // version) to single segment index
        MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
        IndexWriterConfig iwc = new IndexWriterConfig(null).setMergePolicy(mp);
        IndexWriter w = new IndexWriter(dir, iwc);
        w.addIndexes(ramDir);
        try {
            w.commit();
        } finally {
            w.close();
        }
        // determine count of segments in modified index
        final int origSegCount = getNumberOfSegments(dir);
        // ensure there is only one commit
        assertEquals(1, DirectoryReader.listCommits(dir).size());
        newIndexUpgrader(dir).upgrade();
        final int segCount = checkAllSegmentsUpgraded(dir, indexCreatedVersion);
        assertEquals("Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged", origSegCount, segCount);
        dir.close();
    }
}

Example 18 with MockAnalyzer

use of org.apache.lucene.analysis.MockAnalyzer in project lucene-solr by apache.

the class TestBackwardsCompatibility method testAddOldIndexes.

public void testAddOldIndexes() throws IOException {
    for (String name : oldNames) {
        if (VERBOSE) {
            System.out.println("\nTEST: old index " + name);
        }
        Directory oldDir = oldIndexDirs.get(name);
        SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
        Directory targetDir = newDirectory();
        if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
            // both indexes are not compatible
            Directory targetDir2 = newDirectory();
            IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> w.addIndexes(oldDir));
            assertTrue(e.getMessage(), e.getMessage().startsWith("Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));
            w.close();
            targetDir2.close();
            // for the next test, we simulate writing to an index that was created on the same major version
            new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
        }
        IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
        w.addIndexes(oldDir);
        w.close();
        targetDir.close();
        if (VERBOSE) {
            System.out.println("\nTEST: done adding indices; now close");
        }
        targetDir.close();
    }
}

Example 19 with MockAnalyzer

use of org.apache.lucene.analysis.MockAnalyzer in project lucene-solr by apache.

the class TestBackwardsCompatibility method testAddOldIndexesReader.

public void testAddOldIndexesReader() throws IOException {
    for (String name : oldNames) {
        Directory oldDir = oldIndexDirs.get(name);
        SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
        DirectoryReader reader = DirectoryReader.open(oldDir);
        Directory targetDir = newDirectory();
        if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
            Directory targetDir2 = newDirectory();
            IndexWriter w = new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));
            assertEquals(e.getMessage(), "Cannot merge a segment that has been created with major version 6 into this index which has been created by major version 7");
            w.close();
            targetDir2.close();
            // for the next test, we simulate writing to an index that was created on the same major version
            new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
        }
        IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
        TestUtil.addIndexesSlowly(w, reader);
        w.close();
        reader.close();
        targetDir.close();
    }
}

Example 20 with MockAnalyzer

use of org.apache.lucene.analysis.MockAnalyzer in project lucene-solr by apache.

the class TestBackwardsCompatibility method testCreateSortedIndex.

// ant test -Dtestcase=TestBackwardsCompatibility -Dtestmethod=testCreateSortedIndex -Dtests.codec=default -Dtests.useSecurityManager=false -Dtests.bwcdir=/tmp/sorted
public void testCreateSortedIndex() throws Exception {
    Path indexDir = getIndexDir().resolve("sorted");
    Files.deleteIfExists(indexDir);
    Directory dir = newFSDirectory(indexDir);
    LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
    mp.setNoCFSRatio(1.0);
    mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
    MockAnalyzer analyzer = new MockAnalyzer(random());
    analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
    // TODO: remove randomness
    IndexWriterConfig conf = new IndexWriterConfig(analyzer);
    conf.setMergePolicy(mp);
    conf.setUseCompoundFile(false);
    conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));
    IndexWriter writer = new IndexWriter(dir, conf);
    LineFileDocs docs = new LineFileDocs(random());
    SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    parser.setTimeZone(TimeZone.getTimeZone("UTC"));
    ParsePosition position = new ParsePosition(0);
    Field dateDVField = null;
    for (int i = 0; i < 50; i++) {
        Document doc = docs.nextDoc();
        String dateString = doc.get("date");
        position.setIndex(0);
        Date date = parser.parse(dateString, position);
        if (position.getErrorIndex() != -1) {
            throw new AssertionError("failed to parse \"" + dateString + "\" as date");
        }
        if (position.getIndex() != dateString.length()) {
            throw new AssertionError("failed to parse \"" + dateString + "\" as date");
        }
        if (dateDVField == null) {
            dateDVField = new NumericDocValuesField("dateDV", 0l);
            doc.add(dateDVField);
        }
        dateDVField.setLongValue(date.getTime());
        if (i == 250) {
            writer.commit();
        }
        writer.addDocument(doc);
    }
    writer.forceMerge(1);
    writer.close();
    dir.close();
}