Examples with StringField org.apache.lucene.document.StringField used on opensource projects

Example 81 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestMixedDocValuesUpdates method testStressMultiThreading.

public void testStressMultiThreading() throws Exception {
    final Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    final IndexWriter writer = new IndexWriter(dir, conf);
    // create index
    final int numFields = TestUtil.nextInt(random(), 2, 4);
    final int numThreads = TestUtil.nextInt(random(), 3, 6);
    final int numDocs = atLeast(2000);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        doc.add(new StringField("id", "doc" + i, Store.NO));
        double group = random().nextDouble();
        String g;
        if (group < 0.1)
            g = "g0";
        else if (group < 0.5)
            g = "g1";
        else if (group < 0.8)
            g = "g2";
        else
            g = "g3";
        doc.add(new StringField("updKey", g, Store.NO));
        for (int j = 0; j < numFields; j++) {
            long value = random().nextInt();
            doc.add(new BinaryDocValuesField("f" + j, TestBinaryDocValuesUpdates.toBytes(value)));
            // control, always updated to f * 2
            doc.add(new NumericDocValuesField("cf" + j, value * 2));
        }
        writer.addDocument(doc);
    }
    final CountDownLatch done = new CountDownLatch(numThreads);
    final AtomicInteger numUpdates = new AtomicInteger(atLeast(100));
    // same thread updates a field as well as reopens
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread("UpdateThread-" + i) {

            @Override
            public void run() {
                DirectoryReader reader = null;
                boolean success = false;
                try {
                    Random random = random();
                    while (numUpdates.getAndDecrement() > 0) {
                        double group = random.nextDouble();
                        Term t;
                        if (group < 0.1)
                            t = new Term("updKey", "g0");
                        else if (group < 0.5)
                            t = new Term("updKey", "g1");
                        else if (group < 0.8)
                            t = new Term("updKey", "g2");
                        else
                            t = new Term("updKey", "g3");
                        //              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
                        int field = random().nextInt(numFields);
                        final String f = "f" + field;
                        final String cf = "cf" + field;
                        long updValue = random.nextInt();
                        //              System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
                        writer.updateDocValues(t, new BinaryDocValuesField(f, TestBinaryDocValuesUpdates.toBytes(updValue)), new NumericDocValuesField(cf, updValue * 2));
                        if (random.nextDouble() < 0.2) {
                            // delete a random document
                            int doc = random.nextInt(numDocs);
                            //                System.out.println("[" + Thread.currentThread().getName() + "] deleteDoc=doc" + doc);
                            writer.deleteDocuments(new Term("id", "doc" + doc));
                        }
                        if (random.nextDouble() < 0.05) {
                            // commit every 20 updates on average
                            //                  System.out.println("[" + Thread.currentThread().getName() + "] commit");
                            writer.commit();
                        }
                        if (random.nextDouble() < 0.1) {
                            // reopen NRT reader (apply updates), on average once every 10 updates
                            if (reader == null) {
                                //                  System.out.println("[" + Thread.currentThread().getName() + "] open NRT");
                                reader = DirectoryReader.open(writer);
                            } else {
                                //                  System.out.println("[" + Thread.currentThread().getName() + "] reopen NRT");
                                DirectoryReader r2 = DirectoryReader.openIfChanged(reader, writer);
                                if (r2 != null) {
                                    reader.close();
                                    reader = r2;
                                }
                            }
                        }
                    }
                    //            System.out.println("[" + Thread.currentThread().getName() + "] DONE");
                    success = true;
                } catch (IOException e) {
                    throw new RuntimeException(e);
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (IOException e) {
                            if (success) {
                                // suppress this exception only if there was another exception
                                throw new RuntimeException(e);
                            }
                        }
                    }
                    done.countDown();
                }
            }
        };
    }
    for (Thread t : threads) t.start();
    done.await();
    writer.close();
    DirectoryReader reader = DirectoryReader.open(dir);
    BytesRef scratch = new BytesRef();
    for (LeafReaderContext context : reader.leaves()) {
        LeafReader r = context.reader();
        for (int i = 0; i < numFields; i++) {
            BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
            NumericDocValues control = r.getNumericDocValues("cf" + i);
            Bits liveDocs = r.getLiveDocs();
            for (int j = 0; j < r.maxDoc(); j++) {
                if (liveDocs == null || liveDocs.get(j)) {
                    assertEquals(j, control.advance(j));
                    long ctrlValue = control.longValue();
                    assertEquals(j, bdv.advance(j));
                    long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv) * 2;
                    //              if (ctrlValue != bdvValue) {
                    //                System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.document(j).get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
                    //              }
                    assertEquals(ctrlValue, bdvValue);
                }
            }
        }
    }
    reader.close();
    dir.close();
}

Example 82 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestIndexingSequenceNumbers method testStressConcurrentCommit.

@Slow
public void testStressConcurrentCommit() throws Exception {
    final int opCount = atLeast(10000);
    final int idCount = TestUtil.nextInt(random(), 10, 1000);
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    // Cannot use RIW since it randomly commits:
    final IndexWriter w = new IndexWriter(dir, iwc);
    final int numThreads = TestUtil.nextInt(random(), 2, 10);
    Thread[] threads = new Thread[numThreads];
    //System.out.println("TEST: iter=" + iter + " opCount=" + opCount + " idCount=" + idCount + " threadCount=" + threads.length);
    final CountDownLatch startingGun = new CountDownLatch(1);
    List<List<Operation>> threadOps = new ArrayList<>();
    Object commitLock = new Object();
    final List<Operation> commits = new ArrayList<>();
    // the end to verify it reflects the correct updates
    for (int i = 0; i < threads.length; i++) {
        final List<Operation> ops = new ArrayList<>();
        threadOps.add(ops);
        final int threadID = i;
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    startingGun.await();
                    for (int i = 0; i < opCount; i++) {
                        Operation op = new Operation();
                        op.threadID = threadID;
                        if (random().nextInt(500) == 17) {
                            op.what = 2;
                            synchronized (commitLock) {
                                op.seqNo = w.commit();
                                if (op.seqNo != -1) {
                                    commits.add(op);
                                }
                            }
                        } else {
                            op.id = random().nextInt(idCount);
                            Term idTerm = new Term("id", "" + op.id);
                            if (random().nextInt(10) == 1) {
                                op.what = 1;
                                if (random().nextBoolean()) {
                                    op.seqNo = w.deleteDocuments(idTerm);
                                } else {
                                    op.seqNo = w.deleteDocuments(new TermQuery(idTerm));
                                }
                            } else {
                                Document doc = new Document();
                                doc.add(new StoredField("thread", threadID));
                                doc.add(new StringField("id", "" + op.id, Field.Store.NO));
                                if (random().nextBoolean()) {
                                    List<Document> docs = new ArrayList<>();
                                    docs.add(doc);
                                    op.seqNo = w.updateDocuments(idTerm, docs);
                                } else {
                                    op.seqNo = w.updateDocument(idTerm, doc);
                                }
                                op.what = 0;
                            }
                            ops.add(op);
                        }
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        threads[i].start();
    }
    startingGun.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    Operation commitOp = new Operation();
    commitOp.seqNo = w.commit();
    if (commitOp.seqNo != -1) {
        commits.add(commitOp);
    }
    List<IndexCommit> indexCommits = DirectoryReader.listCommits(dir);
    assertEquals(commits.size(), indexCommits.size());
    int[] expectedThreadIDs = new int[idCount];
    long[] seqNos = new long[idCount];
    //System.out.println("TEST: " + commits.size() + " commits");
    for (int i = 0; i < commits.size(); i++) {
        // this commit point should reflect all operations <= this seqNo
        long commitSeqNo = commits.get(i).seqNo;
        //System.out.println("  commit " + i + ": seqNo=" + commitSeqNo + " segs=" + indexCommits.get(i));
        Arrays.fill(expectedThreadIDs, -1);
        Arrays.fill(seqNos, 0);
        for (int threadID = 0; threadID < threadOps.size(); threadID++) {
            long lastSeqNo = 0;
            for (Operation op : threadOps.get(threadID)) {
                if (op.seqNo <= commitSeqNo && op.seqNo > seqNos[op.id]) {
                    seqNos[op.id] = op.seqNo;
                    if (op.what == 0) {
                        expectedThreadIDs[op.id] = threadID;
                    } else {
                        expectedThreadIDs[op.id] = -1;
                    }
                }
                assertTrue(op.seqNo > lastSeqNo);
                lastSeqNo = op.seqNo;
            }
        }
        DirectoryReader r = DirectoryReader.open(indexCommits.get(i));
        IndexSearcher s = new IndexSearcher(r);
        for (int id = 0; id < idCount; id++) {
            //System.out.println("TEST: check id=" + id + " expectedThreadID=" + expectedThreadIDs[id]);
            TopDocs hits = s.search(new TermQuery(new Term("id", "" + id)), 1);
            if (expectedThreadIDs[id] != -1) {
                assertEquals(1, hits.totalHits);
                Document doc = r.document(hits.scoreDocs[0].doc);
                int actualThreadID = doc.getField("thread").numericValue().intValue();
                if (expectedThreadIDs[id] != actualThreadID) {
                    System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs actualThreadID=" + actualThreadID + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
                    for (int threadID = 0; threadID < threadOps.size(); threadID++) {
                        for (Operation op : threadOps.get(threadID)) {
                            if (id == op.id) {
                                System.out.println("  threadID=" + threadID + " seqNo=" + op.seqNo + " " + (op.what == 2 ? "updated" : "deleted"));
                            }
                        }
                    }
                    assertEquals("id=" + id, expectedThreadIDs[id], actualThreadID);
                }
            } else if (hits.totalHits != 0) {
                System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs totalHits=" + hits.totalHits + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
                for (int threadID = 0; threadID < threadOps.size(); threadID++) {
                    for (Operation op : threadOps.get(threadID)) {
                        if (id == op.id) {
                            System.out.println("  threadID=" + threadID + " seqNo=" + op.seqNo + " " + (op.what == 2 ? "updated" : "del"));
                        }
                    }
                }
                assertEquals(0, hits.totalHits);
            }
        }
        w.close();
        r.close();
    }
    dir.close();
}

Example 83 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestIndexingSequenceNumbers method testStressConcurrentAddAndDeleteAndCommit.

@Slow
public void testStressConcurrentAddAndDeleteAndCommit() throws Exception {
    final int opCount = atLeast(10000);
    final int idCount = TestUtil.nextInt(random(), 10, 1000);
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    // Cannot use RIW since it randomly commits:
    final IndexWriter w = new IndexWriter(dir, iwc);
    final int numThreads = TestUtil.nextInt(random(), 2, 5);
    Thread[] threads = new Thread[numThreads];
    //System.out.println("TEST: iter=" + iter + " opCount=" + opCount + " idCount=" + idCount + " threadCount=" + threads.length);
    final CountDownLatch startingGun = new CountDownLatch(1);
    List<List<Operation>> threadOps = new ArrayList<>();
    Object commitLock = new Object();
    final List<Operation> commits = new ArrayList<>();
    // multiple threads update the same set of documents, and we randomly commit
    for (int i = 0; i < threads.length; i++) {
        final List<Operation> ops = new ArrayList<>();
        threadOps.add(ops);
        final int threadID = i;
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    startingGun.await();
                    for (int i = 0; i < opCount; i++) {
                        Operation op = new Operation();
                        op.threadID = threadID;
                        if (random().nextInt(500) == 17) {
                            op.what = 2;
                            synchronized (commitLock) {
                                op.seqNo = w.commit();
                                if (op.seqNo != -1) {
                                    commits.add(op);
                                }
                            }
                        } else {
                            op.id = random().nextInt(idCount);
                            Term idTerm = new Term("id", "" + op.id);
                            if (random().nextInt(10) == 1) {
                                op.what = 1;
                                if (random().nextBoolean()) {
                                    op.seqNo = w.deleteDocuments(idTerm);
                                } else {
                                    op.seqNo = w.deleteDocuments(new TermQuery(idTerm));
                                }
                            } else {
                                Document doc = new Document();
                                doc.add(new StoredField("threadop", threadID + "-" + ops.size()));
                                doc.add(new StringField("id", "" + op.id, Field.Store.NO));
                                if (random().nextBoolean()) {
                                    List<Document> docs = new ArrayList<>();
                                    docs.add(doc);
                                    op.seqNo = w.addDocuments(docs);
                                } else {
                                    op.seqNo = w.addDocument(doc);
                                }
                                op.what = 3;
                            }
                            ops.add(op);
                        }
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        threads[i].setName("thread" + threadID);
        threads[i].start();
    }
    startingGun.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    Operation commitOp = new Operation();
    commitOp.seqNo = w.commit();
    if (commitOp.seqNo != -1) {
        commits.add(commitOp);
    }
    List<IndexCommit> indexCommits = DirectoryReader.listCommits(dir);
    assertEquals(commits.size(), indexCommits.size());
    // how many docs with this id are expected:
    int[] expectedCounts = new int[idCount];
    long[] lastDelSeqNos = new long[idCount];
    //System.out.println("TEST: " + commits.size() + " commits");
    for (int i = 0; i < commits.size(); i++) {
        // this commit point should reflect all operations <= this seqNo
        long commitSeqNo = commits.get(i).seqNo;
        //System.out.println("  commit " + i + ": seqNo=" + commitSeqNo + " segs=" + indexCommits.get(i));
        // first find the highest seqNo of the last delete op, for each id, prior to this commit:
        Arrays.fill(lastDelSeqNos, -1);
        for (int threadID = 0; threadID < threadOps.size(); threadID++) {
            long lastSeqNo = 0;
            for (Operation op : threadOps.get(threadID)) {
                if (op.what == 1 && op.seqNo <= commitSeqNo && op.seqNo > lastDelSeqNos[op.id]) {
                    lastDelSeqNos[op.id] = op.seqNo;
                }
                // within one thread the seqNos must only increase:
                assertTrue(op.seqNo > lastSeqNo);
                lastSeqNo = op.seqNo;
            }
        }
        // then count how many adds happened since the last delete and before this commit:
        Arrays.fill(expectedCounts, 0);
        for (int threadID = 0; threadID < threadOps.size(); threadID++) {
            for (Operation op : threadOps.get(threadID)) {
                if (op.what == 3 && op.seqNo <= commitSeqNo && op.seqNo > lastDelSeqNos[op.id]) {
                    expectedCounts[op.id]++;
                }
            }
        }
        DirectoryReader r = DirectoryReader.open(indexCommits.get(i));
        IndexSearcher s = new IndexSearcher(r);
        for (int id = 0; id < idCount; id++) {
            //System.out.println("TEST: check id=" + id + " expectedThreadID=" + expectedThreadIDs[id]);
            int actualCount = s.count(new TermQuery(new Term("id", "" + id)));
            if (expectedCounts[id] != actualCount) {
                System.out.println("TEST: FAIL r=" + r + " id=" + id + " commitSeqNo=" + commitSeqNo);
                for (int threadID = 0; threadID < threadOps.size(); threadID++) {
                    int opCount2 = 0;
                    for (Operation op : threadOps.get(threadID)) {
                        if (op.id == id) {
                            boolean shouldCount = op.seqNo <= commitSeqNo && op.seqNo > lastDelSeqNos[op.id];
                            System.out.println("  id=" + id + " what=" + op.what + " threadop=" + threadID + "-" + opCount2 + " seqNo=" + op.seqNo + " vs lastDelSeqNo=" + lastDelSeqNos[op.id] + " shouldCount=" + shouldCount);
                        }
                        opCount2++;
                    }
                }
                TopDocs hits = s.search(new TermQuery(new Term("id", "" + id)), 1 + actualCount);
                for (ScoreDoc hit : hits.scoreDocs) {
                    System.out.println("  hit: " + s.doc(hit.doc).get("threadop"));
                }
                for (LeafReaderContext ctx : r.leaves()) {
                    System.out.println("  sub=" + ctx.reader());
                    Bits liveDocs = ctx.reader().getLiveDocs();
                    for (int docID = 0; docID < ctx.reader().maxDoc(); docID++) {
                        System.out.println("    docID=" + docID + " threadop=" + ctx.reader().document(docID).get("threadop") + (liveDocs != null && liveDocs.get(docID) == false ? " (deleted)" : ""));
                    }
                }
                assertEquals("commit " + i + " of " + commits.size() + " id=" + id + " reader=" + r, expectedCounts[id], actualCount);
            }
        }
        w.close();
        r.close();
    }
    dir.close();
}

Example 84 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestMixedDocValuesUpdates method testManyReopensAndFields.

public void testManyReopensAndFields() throws Exception {
    Directory dir = newDirectory();
    final Random random = random();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
    LogMergePolicy lmp = newLogMergePolicy();
    // merge often
    lmp.setMergeFactor(3);
    conf.setMergePolicy(lmp);
    IndexWriter writer = new IndexWriter(dir, conf);
    final boolean isNRT = random.nextBoolean();
    DirectoryReader reader;
    if (isNRT) {
        reader = DirectoryReader.open(writer);
    } else {
        writer.commit();
        reader = DirectoryReader.open(dir);
    }
    // 3-7
    final int numFields = random.nextInt(4) + 3;
    // 1-3
    final int numNDVFields = random.nextInt(numFields / 2) + 1;
    final long[] fieldValues = new long[numFields];
    for (int i = 0; i < fieldValues.length; i++) {
        fieldValues[i] = 1;
    }
    int numRounds = atLeast(15);
    int docID = 0;
    for (int i = 0; i < numRounds; i++) {
        int numDocs = atLeast(5);
        // System.out.println("TEST: round=" + i + ", numDocs=" + numDocs);
        for (int j = 0; j < numDocs; j++) {
            Document doc = new Document();
            doc.add(new StringField("id", "doc-" + docID, Store.NO));
            // update key
            doc.add(new StringField("key", "all", Store.NO));
            // add all fields with their current value
            for (int f = 0; f < fieldValues.length; f++) {
                if (f < numNDVFields) {
                    doc.add(new NumericDocValuesField("f" + f, fieldValues[f]));
                } else {
                    doc.add(new BinaryDocValuesField("f" + f, TestBinaryDocValuesUpdates.toBytes(fieldValues[f])));
                }
            }
            writer.addDocument(doc);
            ++docID;
        }
        int fieldIdx = random.nextInt(fieldValues.length);
        String updateField = "f" + fieldIdx;
        if (fieldIdx < numNDVFields) {
            writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
        } else {
            writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
        }
        if (random.nextDouble() < 0.2) {
            // might also delete an already deleted document, ok!
            int deleteDoc = random.nextInt(docID);
            writer.deleteDocuments(new Term("id", "doc-" + deleteDoc));
        //        System.out.println("[" + Thread.currentThread().getName() + "]: deleted document: doc-" + deleteDoc);
        }
        // verify reader
        if (!isNRT) {
            writer.commit();
        }
        //      System.out.println("[" + Thread.currentThread().getName() + "]: reopen reader: " + reader);
        DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
        assertNotNull(newReader);
        reader.close();
        reader = newReader;
        //      System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
        // we delete at most one document per round
        assertTrue(reader.numDocs() > 0);
        for (LeafReaderContext context : reader.leaves()) {
            LeafReader r = context.reader();
            //        System.out.println(((SegmentReader) r).getSegmentName());
            Bits liveDocs = r.getLiveDocs();
            for (int field = 0; field < fieldValues.length; field++) {
                String f = "f" + field;
                BinaryDocValues bdv = r.getBinaryDocValues(f);
                NumericDocValues ndv = r.getNumericDocValues(f);
                if (field < numNDVFields) {
                    assertNotNull(ndv);
                    assertNull(bdv);
                } else {
                    assertNull(ndv);
                    assertNotNull(bdv);
                }
                int maxDoc = r.maxDoc();
                for (int doc = 0; doc < maxDoc; doc++) {
                    if (liveDocs == null || liveDocs.get(doc)) {
                        //              System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
                        if (field < numNDVFields) {
                            assertEquals(doc, ndv.advance(doc));
                            assertEquals("invalid numeric value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.longValue());
                        } else {
                            assertEquals(doc, bdv.advance(doc));
                            assertEquals("invalid binary value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv));
                        }
                    }
                }
            }
        }
    //      System.out.println();
    }
    writer.close();
    IOUtils.close(reader, dir);
}

Example 85 with StringField

use of org.apache.lucene.document.StringField in project lucene-solr by apache.

the class TestMultiTermsEnum method testNoTermsInField.

// LUCENE-6826
public void testNoTermsInField() throws Exception {
    Directory directory = new RAMDirectory();
    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())));
    Document document = new Document();
    document.add(new StringField("deleted", "0", Field.Store.YES));
    writer.addDocument(document);
    DirectoryReader reader = DirectoryReader.open(writer);
    writer.close();
    Directory directory2 = new RAMDirectory();
    writer = new IndexWriter(directory2, new IndexWriterConfig(new MockAnalyzer(random())));
    List<LeafReaderContext> leaves = reader.leaves();
    CodecReader[] codecReaders = new CodecReader[leaves.size()];
    for (int i = 0; i < leaves.size(); i++) {
        codecReaders[i] = new MigratingCodecReader((CodecReader) leaves.get(i).reader());
    }
    // <- bang
    writer.addIndexes(codecReaders);
    IOUtils.close(writer, reader, directory);
}