Examples with BinaryDocValuesField org.apache.lucene.document.BinaryDocValuesField used on opensource projects

Example 16 with BinaryDocValuesField

use of org.apache.lucene.document.BinaryDocValuesField in project lucene-solr by apache.

the class TestDocValuesIndexing method testSameFieldNameForPostingAndDocValue.

public void testSameFieldNameForPostingAndDocValue() throws Exception {
    // LUCENE-5192: FieldInfos.Builder neglected to update
    // globalFieldNumbers.docValuesType map if the field existed, resulting in
    // potentially adding the same field with different DV types.
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter writer = new IndexWriter(dir, conf);
    Document doc = new Document();
    doc.add(new StringField("f", "mock-value", Store.NO));
    doc.add(new NumericDocValuesField("f", 5));
    writer.addDocument(doc);
    writer.commit();
    Document doc2 = new Document();
    doc2.add(new BinaryDocValuesField("f", new BytesRef("mock")));
    expectThrows(IllegalArgumentException.class, () -> {
        writer.addDocument(doc2);
    });
    writer.rollback();
    dir.close();
}

Example 17 with BinaryDocValuesField

use of org.apache.lucene.document.BinaryDocValuesField in project lucene-solr by apache.

the class TestDocValuesIndexing method testMixedTypesDifferentThreads.

// Two documents with same field as different types, added
// from separate threads:
public void testMixedTypesDifferentThreads() throws Exception {
    Directory dir = newDirectory();
    final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    final CountDownLatch startingGun = new CountDownLatch(1);
    final AtomicBoolean hitExc = new AtomicBoolean();
    Thread[] threads = new Thread[3];
    for (int i = 0; i < 3; i++) {
        Field field;
        if (i == 0) {
            field = new SortedDocValuesField("foo", new BytesRef("hello"));
        } else if (i == 1) {
            field = new NumericDocValuesField("foo", 0);
        } else {
            field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
        }
        final Document doc = new Document();
        doc.add(field);
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    startingGun.await();
                    w.addDocument(doc);
                } catch (IllegalArgumentException iae) {
                    // expected
                    hitExc.set(true);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        threads[i].start();
    }
    startingGun.countDown();
    for (Thread t : threads) {
        t.join();
    }
    assertTrue(hitExc.get());
    w.close();
    dir.close();
}

Example 18 with BinaryDocValuesField

use of org.apache.lucene.document.BinaryDocValuesField in project lucene-solr by apache.

the class TestDocValuesIndexing method testMixedTypesAfterReopenAppend2.

public void testMixedTypesAfterReopenAppend2() throws IOException {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
    w.addDocument(doc);
    w.close();
    Document doc2 = new Document();
    IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    doc2.add(new StringField("foo", "bar", Field.Store.NO));
    doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
    // NOTE: this case follows a different code path inside
    // DefaultIndexingChain/FieldInfos, because the field (foo)
    // is first added without DocValues:
    expectThrows(IllegalArgumentException.class, () -> {
        w2.addDocument(doc2);
    });
    w2.forceMerge(1);
    w2.close();
    dir.close();
}

Example 19 with BinaryDocValuesField

use of org.apache.lucene.document.BinaryDocValuesField in project lucene-solr by apache.

the class TestMixedDocValuesUpdates method testStressMultiThreading.

public void testStressMultiThreading() throws Exception {
    final Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    final IndexWriter writer = new IndexWriter(dir, conf);
    // create index
    final int numFields = TestUtil.nextInt(random(), 2, 4);
    final int numThreads = TestUtil.nextInt(random(), 3, 6);
    final int numDocs = atLeast(2000);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        doc.add(new StringField("id", "doc" + i, Store.NO));
        double group = random().nextDouble();
        String g;
        if (group < 0.1)
            g = "g0";
        else if (group < 0.5)
            g = "g1";
        else if (group < 0.8)
            g = "g2";
        else
            g = "g3";
        doc.add(new StringField("updKey", g, Store.NO));
        for (int j = 0; j < numFields; j++) {
            long value = random().nextInt();
            doc.add(new BinaryDocValuesField("f" + j, TestBinaryDocValuesUpdates.toBytes(value)));
            // control, always updated to f * 2
            doc.add(new NumericDocValuesField("cf" + j, value * 2));
        }
        writer.addDocument(doc);
    }
    final CountDownLatch done = new CountDownLatch(numThreads);
    final AtomicInteger numUpdates = new AtomicInteger(atLeast(100));
    // same thread updates a field as well as reopens
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread("UpdateThread-" + i) {

            @Override
            public void run() {
                DirectoryReader reader = null;
                boolean success = false;
                try {
                    Random random = random();
                    while (numUpdates.getAndDecrement() > 0) {
                        double group = random.nextDouble();
                        Term t;
                        if (group < 0.1)
                            t = new Term("updKey", "g0");
                        else if (group < 0.5)
                            t = new Term("updKey", "g1");
                        else if (group < 0.8)
                            t = new Term("updKey", "g2");
                        else
                            t = new Term("updKey", "g3");
                        //              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
                        int field = random().nextInt(numFields);
                        final String f = "f" + field;
                        final String cf = "cf" + field;
                        long updValue = random.nextInt();
                        //              System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
                        writer.updateDocValues(t, new BinaryDocValuesField(f, TestBinaryDocValuesUpdates.toBytes(updValue)), new NumericDocValuesField(cf, updValue * 2));
                        if (random.nextDouble() < 0.2) {
                            // delete a random document
                            int doc = random.nextInt(numDocs);
                            //                System.out.println("[" + Thread.currentThread().getName() + "] deleteDoc=doc" + doc);
                            writer.deleteDocuments(new Term("id", "doc" + doc));
                        }
                        if (random.nextDouble() < 0.05) {
                            // commit every 20 updates on average
                            //                  System.out.println("[" + Thread.currentThread().getName() + "] commit");
                            writer.commit();
                        }
                        if (random.nextDouble() < 0.1) {
                            // reopen NRT reader (apply updates), on average once every 10 updates
                            if (reader == null) {
                                //                  System.out.println("[" + Thread.currentThread().getName() + "] open NRT");
                                reader = DirectoryReader.open(writer);
                            } else {
                                //                  System.out.println("[" + Thread.currentThread().getName() + "] reopen NRT");
                                DirectoryReader r2 = DirectoryReader.openIfChanged(reader, writer);
                                if (r2 != null) {
                                    reader.close();
                                    reader = r2;
                                }
                            }
                        }
                    }
                    //            System.out.println("[" + Thread.currentThread().getName() + "] DONE");
                    success = true;
                } catch (IOException e) {
                    throw new RuntimeException(e);
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (IOException e) {
                            if (success) {
                                // suppress this exception only if there was another exception
                                throw new RuntimeException(e);
                            }
                        }
                    }
                    done.countDown();
                }
            }
        };
    }
    for (Thread t : threads) t.start();
    done.await();
    writer.close();
    DirectoryReader reader = DirectoryReader.open(dir);
    BytesRef scratch = new BytesRef();
    for (LeafReaderContext context : reader.leaves()) {
        LeafReader r = context.reader();
        for (int i = 0; i < numFields; i++) {
            BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
            NumericDocValues control = r.getNumericDocValues("cf" + i);
            Bits liveDocs = r.getLiveDocs();
            for (int j = 0; j < r.maxDoc(); j++) {
                if (liveDocs == null || liveDocs.get(j)) {
                    assertEquals(j, control.advance(j));
                    long ctrlValue = control.longValue();
                    assertEquals(j, bdv.advance(j));
                    long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv) * 2;
                    //              if (ctrlValue != bdvValue) {
                    //                System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.document(j).get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
                    //              }
                    assertEquals(ctrlValue, bdvValue);
                }
            }
        }
    }
    reader.close();
    dir.close();
}

Example 20 with BinaryDocValuesField

use of org.apache.lucene.document.BinaryDocValuesField in project lucene-solr by apache.

the class TestMultiDocValues method testBinary.

public void testBinary() throws Exception {
    Directory dir = newDirectory();
    Document doc = new Document();
    Field field = new BinaryDocValuesField("bytes", new BytesRef());
    doc.add(field);
    IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
    iwc.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
    for (int i = 0; i < numDocs; i++) {
        BytesRef ref = new BytesRef(TestUtil.randomUnicodeString(random()));
        field.setBytesValue(ref);
        iw.addDocument(doc);
        if (random().nextInt(17) == 0) {
            iw.commit();
        }
    }
    DirectoryReader ir = iw.getReader();
    iw.forceMerge(1);
    DirectoryReader ir2 = iw.getReader();
    LeafReader merged = getOnlyLeafReader(ir2);
    iw.close();
    BinaryDocValues multi = MultiDocValues.getBinaryValues(ir, "bytes");
    BinaryDocValues single = merged.getBinaryDocValues("bytes");
    for (int i = 0; i < numDocs; i++) {
        assertEquals(i, multi.nextDoc());
        assertEquals(i, single.nextDoc());
        final BytesRef expected = BytesRef.deepCopyOf(single.binaryValue());
        final BytesRef actual = multi.binaryValue();
        assertEquals(expected, actual);
    }
    testRandomAdvance(merged.getBinaryDocValues("bytes"), MultiDocValues.getBinaryValues(ir, "bytes"));
    testRandomAdvanceExact(merged.getBinaryDocValues("bytes"), MultiDocValues.getBinaryValues(ir, "bytes"), merged.maxDoc());
    ir.close();
    ir2.close();
    dir.close();
}