Examples with LineFileDocs org.apache.lucene.util.LineFileDocs used on opensource projects

Example 31 with LineFileDocs

use of org.apache.lucene.util.LineFileDocs in project lucene-solr by apache.

the class TestNRTReplication method testCrashPrimaryWhileCopying.

@Nightly
public void testCrashPrimaryWhileCopying() throws Exception {
    Path path1 = createTempDir("1");
    NodeProcess primary = startNode(-1, 0, path1, -1, true);
    Path path2 = createTempDir("2");
    NodeProcess replica = startNode(primary.tcpPort, 1, path2, -1, true);
    sendReplicasToPrimary(primary, replica);
    // Index 100 docs into primary:
    LineFileDocs docs = new LineFileDocs(random());
    try (Connection c = new Connection(primary.tcpPort)) {
        c.out.writeByte(SimplePrimaryNode.CMD_INDEXING);
        for (int i = 0; i < 100; i++) {
            Document doc = docs.nextDoc();
            primary.addOrUpdateDocument(c, doc, false);
        }
    }
    // Refresh primary, which also pushes (async) to replica:
    long primaryVersion1 = primary.flush(0);
    assertTrue(primaryVersion1 > 0);
    Thread.sleep(TestUtil.nextInt(random(), 1, 30));
    // Crash primary, likely/hopefully while replica is still copying
    primary.crash();
    // Could see either 100 docs (replica finished before crash) or 0 docs:
    try (Connection c = new Connection(replica.tcpPort)) {
        c.out.writeByte(SimplePrimaryNode.CMD_SEARCH_ALL);
        c.flush();
        long version = c.in.readVLong();
        int hitCount = c.in.readVInt();
        if (version == 0) {
            assertEquals(0, hitCount);
        } else {
            assertEquals(primaryVersion1, version);
            assertEquals(100, hitCount);
        }
    }
    primary.close();
    replica.close();
}

Example 32 with LineFileDocs

use of org.apache.lucene.util.LineFileDocs in project lucene-solr by apache.

the class TestNRTReplication method testReplicaCrashNoCommit.

// Start up, index 10 docs, replicate, but crash and restart the replica without committing it:
@Nightly
public void testReplicaCrashNoCommit() throws Exception {
    Path primaryPath = createTempDir("primary");
    NodeProcess primary = startNode(-1, 0, primaryPath, -1, false);
    Path replicaPath = createTempDir("replica");
    NodeProcess replica = startNode(primary.tcpPort, 1, replicaPath, -1, true);
    sendReplicasToPrimary(primary, replica);
    // Index 10 docs into primary:
    LineFileDocs docs = new LineFileDocs(random());
    try (Connection c = new Connection(primary.tcpPort)) {
        c.out.writeByte(SimplePrimaryNode.CMD_INDEXING);
        for (int i = 0; i < 10; i++) {
            Document doc = docs.nextDoc();
            primary.addOrUpdateDocument(c, doc, false);
        }
    }
    // Refresh primary, which also pushes to replica:
    long primaryVersion1 = primary.flush(0);
    assertTrue(primaryVersion1 > 0);
    // Wait for replica to sync up:
    waitForVersionAndHits(replica, primaryVersion1, 10);
    // Crash replica:
    replica.crash();
    // Restart replica:
    replica = startNode(primary.tcpPort, 1, replicaPath, -1, false);
    // On startup the replica searches the last commit (empty here):
    assertVersionAndHits(replica, 0, 0);
    // Ask replica to sync:
    replica.newNRTPoint(primaryVersion1, 0, primary.tcpPort);
    waitForVersionAndHits(replica, primaryVersion1, 10);
    replica.close();
    primary.close();
}

Example 33 with LineFileDocs

use of org.apache.lucene.util.LineFileDocs in project lucene-solr by apache.

the class TestNRTReplication method testCrashReplica.

public void testCrashReplica() throws Exception {
    Path path1 = createTempDir("1");
    NodeProcess primary = startNode(-1, 0, path1, -1, true);
    Path path2 = createTempDir("2");
    NodeProcess replica = startNode(primary.tcpPort, 1, path2, -1, true);
    assertWriteLockHeld(path2);
    sendReplicasToPrimary(primary, replica);
    // Index 10 docs into primary:
    LineFileDocs docs = new LineFileDocs(random());
    try (Connection c = new Connection(primary.tcpPort)) {
        c.out.writeByte(SimplePrimaryNode.CMD_INDEXING);
        for (int i = 0; i < 10; i++) {
            Document doc = docs.nextDoc();
            primary.addOrUpdateDocument(c, doc, false);
        }
    }
    // Refresh primary, which also pushes to replica:
    long primaryVersion1 = primary.flush(0);
    assertTrue(primaryVersion1 > 0);
    // Wait for replica to sync up:
    waitForVersionAndHits(replica, primaryVersion1, 10);
    // Crash replica
    replica.crash();
    sendReplicasToPrimary(primary);
    // Lots of new flushes while replica is down:
    long primaryVersion2 = 0;
    for (int iter = 0; iter < 10; iter++) {
        // Index 10 docs into primary:
        try (Connection c = new Connection(primary.tcpPort)) {
            c.out.writeByte(SimplePrimaryNode.CMD_INDEXING);
            for (int i = 0; i < 10; i++) {
                Document doc = docs.nextDoc();
                primary.addOrUpdateDocument(c, doc, false);
            }
        }
        primaryVersion2 = primary.flush(0);
    }
    // Start up replica again:
    replica = startNode(primary.tcpPort, 1, path2, -1, true);
    sendReplicasToPrimary(primary, replica);
    // Now ask replica to sync:
    replica.newNRTPoint(primaryVersion2, 0, primary.tcpPort);
    // Make sure it sees all docs that were indexed while it was down:
    assertVersionAndHits(primary, primaryVersion2, 110);
    replica.close();
    primary.close();
}

Example 34 with LineFileDocs

use of org.apache.lucene.util.LineFileDocs in project lucene-solr by apache.

the class BaseTokenStreamTestCase method checkRandomData.

private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
    final LineFileDocs docs = new LineFileDocs(random);
    Document doc = null;
    Field field = null, currentField = null;
    StringReader bogus = new StringReader("");
    if (iw != null) {
        doc = new Document();
        FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
        if (random.nextBoolean()) {
            ft.setStoreTermVectors(true);
            ft.setStoreTermVectorOffsets(random.nextBoolean());
            ft.setStoreTermVectorPositions(random.nextBoolean());
            if (ft.storeTermVectorPositions()) {
                ft.setStoreTermVectorPayloads(random.nextBoolean());
            }
        }
        if (random.nextBoolean()) {
            ft.setOmitNorms(true);
        }
        switch(random.nextInt(4)) {
            case 0:
                ft.setIndexOptions(IndexOptions.DOCS);
                break;
            case 1:
                ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
                break;
            case 2:
                ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
                break;
            default:
                if (offsetsAreCorrect) {
                    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
                } else {
                    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
                }
        }
        currentField = field = new Field("dummy", bogus, ft);
        doc.add(currentField);
    }
    try {
        for (int i = 0; i < iterations; i++) {
            String text;
            if (random.nextInt(10) == 7) {
                // real data from linedocs
                text = docs.nextDoc().get("body");
                if (text.length() > maxWordLength) {
                    // Take a random slice from the text...:
                    int startPos = random.nextInt(text.length() - maxWordLength);
                    if (startPos > 0 && Character.isLowSurrogate(text.charAt(startPos))) {
                        // Take care not to split up a surrogate pair:
                        startPos--;
                        assert Character.isHighSurrogate(text.charAt(startPos));
                    }
                    int endPos = startPos + maxWordLength - 1;
                    if (Character.isHighSurrogate(text.charAt(endPos))) {
                        // Take care not to split up a surrogate pair:
                        endPos--;
                    }
                    text = text.substring(startPos, 1 + endPos);
                }
            } else {
                // synthetic
                text = TestUtil.randomAnalysisString(random, maxWordLength, simple);
            }
            try {
                checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect, currentField);
                if (iw != null) {
                    if (random.nextInt(7) == 0) {
                        // pile up a multivalued field
                        IndexableFieldType ft = field.fieldType();
                        currentField = new Field("dummy", bogus, ft);
                        doc.add(currentField);
                    } else {
                        iw.addDocument(doc);
                        if (doc.getFields().size() > 1) {
                            // back to 1 field
                            currentField = field;
                            doc.removeFields("dummy");
                            doc.add(currentField);
                        }
                    }
                }
            } catch (Throwable t) {
                // TODO: really we should pass a random seed to
                // checkAnalysisConsistency then print it here too:
                System.err.println("TEST FAIL: useCharFilter=" + useCharFilter + " text='" + escape(text) + "'");
                Rethrow.rethrow(t);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(docs);
    }
}

Example 35 with LineFileDocs

use of org.apache.lucene.util.LineFileDocs in project lucene-solr by apache.

the class TestAllFilesDetectTruncation method test.

public void test() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    conf.setCodec(TestUtil.getDefaultCodec());
    // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too:
    if (random().nextInt(5) != 1) {
        conf.setUseCompoundFile(false);
        conf.getMergePolicy().setNoCFSRatio(0.0);
    }
    RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
    // Use LineFileDocs so we (hopefully) get most Lucene features
    // tested, e.g. IntPoint was recently added to it:
    LineFileDocs docs = new LineFileDocs(random());
    for (int i = 0; i < 100; i++) {
        riw.addDocument(docs.nextDoc());
        if (random().nextInt(7) == 0) {
            riw.commit();
        }
        if (random().nextInt(20) == 0) {
            riw.deleteDocuments(new Term("docid", Integer.toString(i)));
        }
        if (random().nextInt(15) == 0) {
            riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
        }
    }
    if (TEST_NIGHTLY == false) {
        riw.forceMerge(1);
    }
    riw.close();
    checkTruncation(dir);
    dir.close();
}