Examples with BinaryPoint org.apache.lucene.document.BinaryPoint used on opensource projects

Example 11 with BinaryPoint

use of org.apache.lucene.document.BinaryPoint in project lucene-solr by apache.

the class TestPointValues method testIllegalTooManyBytes.

public void testIllegalTooManyBytes() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    expectThrows(IllegalArgumentException.class, () -> {
        doc.add(new BinaryPoint("dim", new byte[PointValues.MAX_NUM_BYTES + 1]));
    });
    Document doc2 = new Document();
    doc2.add(new IntPoint("dim", 17));
    w.addDocument(doc2);
    w.close();
    dir.close();
}

Example 12 with BinaryPoint

use of org.apache.lucene.document.BinaryPoint in project lucene-solr by apache.

the class TestPointQueries method verifyLongs.

// verify for long values
private static void verifyLongs(long[] values, int[] ids) throws Exception {
    IndexWriterConfig iwc = newIndexWriterConfig();
    // Else we can get O(N^2) merging:
    int mbd = iwc.getMaxBufferedDocs();
    if (mbd != -1 && mbd < values.length / 100) {
        iwc.setMaxBufferedDocs(values.length / 100);
    }
    iwc.setCodec(getCodec());
    Directory dir;
    if (values.length > 100000) {
        dir = newMaybeVirusCheckingFSDirectory(createTempDir("TestRangeTree"));
    } else {
        dir = newMaybeVirusCheckingDirectory();
    }
    int missingPct = random().nextInt(100);
    int deletedPct = random().nextInt(100);
    if (VERBOSE) {
        System.out.println("  missingPct=" + missingPct);
        System.out.println("  deletedPct=" + deletedPct);
    }
    BitSet missing = new BitSet();
    BitSet deleted = new BitSet();
    Document doc = null;
    int lastID = -1;
    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
    for (int ord = 0; ord < values.length; ord++) {
        int id;
        if (ids == null) {
            id = ord;
        } else {
            id = ids[ord];
        }
        if (id != lastID) {
            if (random().nextInt(100) < missingPct) {
                missing.set(id);
                if (VERBOSE) {
                    System.out.println("  missing id=" + id);
                }
            }
            if (doc != null) {
                w.addDocument(doc);
                if (random().nextInt(100) < deletedPct) {
                    int idToDelete = random().nextInt(id);
                    w.deleteDocuments(new Term("id", "" + idToDelete));
                    deleted.set(idToDelete);
                    if (VERBOSE) {
                        System.out.println("  delete id=" + idToDelete);
                    }
                }
            }
            doc = new Document();
            doc.add(newStringField("id", "" + id, Field.Store.NO));
            doc.add(new NumericDocValuesField("id", id));
            lastID = id;
        }
        if (missing.get(id) == false) {
            doc.add(new LongPoint("sn_value", values[id]));
            byte[] bytes = new byte[8];
            NumericUtils.longToSortableBytes(values[id], bytes, 0);
            doc.add(new BinaryPoint("ss_value", bytes));
        }
    }
    w.addDocument(doc);
    if (random().nextBoolean()) {
        if (VERBOSE) {
            System.out.println("  forceMerge(1)");
        }
        w.forceMerge(1);
    }
    final IndexReader r = w.getReader();
    w.close();
    IndexSearcher s = newSearcher(r, false);
    int numThreads = TestUtil.nextInt(random(), 2, 5);
    if (VERBOSE) {
        System.out.println("TEST: use " + numThreads + " query threads; searcher=" + s);
    }
    List<Thread> threads = new ArrayList<>();
    final int iters = atLeast(100);
    final CountDownLatch startingGun = new CountDownLatch(1);
    final AtomicBoolean failed = new AtomicBoolean();
    for (int i = 0; i < numThreads; i++) {
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    _run();
                } catch (Exception e) {
                    failed.set(true);
                    throw new RuntimeException(e);
                }
            }

            private void _run() throws Exception {
                startingGun.await();
                for (int iter = 0; iter < iters && failed.get() == false; iter++) {
                    Long lower = randomValue();
                    Long upper = randomValue();
                    if (upper < lower) {
                        long x = lower;
                        lower = upper;
                        upper = x;
                    }
                    Query query;
                    if (VERBOSE) {
                        System.out.println("\n" + Thread.currentThread().getName() + ": TEST: iter=" + iter + " value=" + lower + " TO " + upper);
                        byte[] tmp = new byte[8];
                        if (lower != null) {
                            NumericUtils.longToSortableBytes(lower, tmp, 0);
                            System.out.println("  lower bytes=" + Arrays.toString(tmp));
                        }
                        if (upper != null) {
                            NumericUtils.longToSortableBytes(upper, tmp, 0);
                            System.out.println("  upper bytes=" + Arrays.toString(tmp));
                        }
                    }
                    if (random().nextBoolean()) {
                        query = LongPoint.newRangeQuery("sn_value", lower, upper);
                    } else {
                        byte[] lowerBytes = new byte[8];
                        NumericUtils.longToSortableBytes(lower, lowerBytes, 0);
                        byte[] upperBytes = new byte[8];
                        NumericUtils.longToSortableBytes(upper, upperBytes, 0);
                        query = BinaryPoint.newRangeQuery("ss_value", lowerBytes, upperBytes);
                    }
                    if (VERBOSE) {
                        System.out.println(Thread.currentThread().getName() + ":  using query: " + query);
                    }
                    final BitSet hits = new BitSet();
                    s.search(query, new SimpleCollector() {

                        private int docBase;

                        @Override
                        public boolean needsScores() {
                            return false;
                        }

                        @Override
                        protected void doSetNextReader(LeafReaderContext context) throws IOException {
                            docBase = context.docBase;
                        }

                        @Override
                        public void collect(int doc) {
                            hits.set(docBase + doc);
                        }
                    });
                    if (VERBOSE) {
                        System.out.println(Thread.currentThread().getName() + ":  hitCount: " + hits.cardinality());
                    }
                    NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
                    for (int docID = 0; docID < r.maxDoc(); docID++) {
                        assertEquals(docID, docIDToID.nextDoc());
                        int id = (int) docIDToID.longValue();
                        boolean expected = missing.get(id) == false && deleted.get(id) == false && values[id] >= lower && values[id] <= upper;
                        if (hits.get(docID) != expected) {
                            // We do exact quantized comparison so the bbox query should never disagree:
                            fail(Thread.currentThread().getName() + ": iter=" + iter + " id=" + id + " docID=" + docID + " value=" + values[id] + " (range: " + lower + " TO " + upper + ") expected " + expected + " but got: " + hits.get(docID) + " deleted?=" + deleted.get(id) + " query=" + query);
                        }
                    }
                }
            }
        };
        thread.setName("T" + i);
        thread.start();
        threads.add(thread);
    }
    startingGun.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    IOUtils.close(r, dir);
}

Example 13 with BinaryPoint

use of org.apache.lucene.document.BinaryPoint in project lucene-solr by apache.

the class TestPointQueries method testPointInSetQueryManyEqualValues.

public void testPointInSetQueryManyEqualValues() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setCodec(getCodec());
    IndexWriter w = new IndexWriter(dir, iwc);
    int zeroCount = 0;
    for (int i = 0; i < 10000; i++) {
        int x = random().nextInt(2);
        if (x == 0) {
            zeroCount++;
        }
        Document doc = new Document();
        doc.add(new IntPoint("int", x));
        doc.add(new LongPoint("long", (long) x));
        doc.add(new FloatPoint("float", (float) x));
        doc.add(new DoublePoint("double", (double) x));
        doc.add(new BinaryPoint("bytes", new byte[] { (byte) x }));
        w.addDocument(doc);
    }
    IndexReader r = DirectoryReader.open(w);
    IndexSearcher s = newSearcher(r, false);
    assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0)));
    assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0, -7)));
    assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 7, 0)));
    assertEquals(10000 - zeroCount, s.count(IntPoint.newSetQuery("int", 1)));
    assertEquals(0, s.count(IntPoint.newSetQuery("int", 2)));
    assertEquals(zeroCount, s.count(LongPoint.newSetQuery("long", 0)));
    assertEquals(zeroCount, s.count(LongPoint.newSetQuery("long", 0, -7)));
    assertEquals(zeroCount, s.count(LongPoint.newSetQuery("long", 7, 0)));
    assertEquals(10000 - zeroCount, s.count(LongPoint.newSetQuery("long", 1)));
    assertEquals(0, s.count(LongPoint.newSetQuery("long", 2)));
    assertEquals(zeroCount, s.count(FloatPoint.newSetQuery("float", 0)));
    assertEquals(zeroCount, s.count(FloatPoint.newSetQuery("float", 0, -7)));
    assertEquals(zeroCount, s.count(FloatPoint.newSetQuery("float", 7, 0)));
    assertEquals(10000 - zeroCount, s.count(FloatPoint.newSetQuery("float", 1)));
    assertEquals(0, s.count(FloatPoint.newSetQuery("float", 2)));
    assertEquals(zeroCount, s.count(DoublePoint.newSetQuery("double", 0)));
    assertEquals(zeroCount, s.count(DoublePoint.newSetQuery("double", 0, -7)));
    assertEquals(zeroCount, s.count(DoublePoint.newSetQuery("double", 7, 0)));
    assertEquals(10000 - zeroCount, s.count(DoublePoint.newSetQuery("double", 1)));
    assertEquals(0, s.count(DoublePoint.newSetQuery("double", 2)));
    assertEquals(zeroCount, s.count(BinaryPoint.newSetQuery("bytes", new byte[] { 0 })));
    assertEquals(zeroCount, s.count(BinaryPoint.newSetQuery("bytes", new byte[] { 0 }, new byte[] { -7 })));
    assertEquals(zeroCount, s.count(BinaryPoint.newSetQuery("bytes", new byte[] { 7 }, new byte[] { 0 })));
    assertEquals(10000 - zeroCount, s.count(BinaryPoint.newSetQuery("bytes", new byte[] { 1 })));
    assertEquals(0, s.count(BinaryPoint.newSetQuery("bytes", new byte[] { 2 })));
    w.close();
    r.close();
    dir.close();
}

Example 14 with BinaryPoint

use of org.apache.lucene.document.BinaryPoint in project lucene-solr by apache.

the class TestPointValues method testIllegalDimChangeOneDoc.

// Illegal schema change tests:
public void testIllegalDimChangeOneDoc() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new BinaryPoint("dim", new byte[4]));
    doc.add(new BinaryPoint("dim", new byte[4], new byte[4]));
    IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
        w.addDocument(doc);
    });
    assertEquals("cannot change point dimension count from 1 to 2 for field=\"dim\"", expected.getMessage());
    w.close();
    dir.close();
}

Example 15 with BinaryPoint

use of org.apache.lucene.document.BinaryPoint in project lucene-solr by apache.

the class TestPointValues method testIllegalNumBytesChangeTwoDocs.

public void testIllegalNumBytesChangeTwoDocs() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new BinaryPoint("dim", new byte[4]));
    w.addDocument(doc);
    Document doc2 = new Document();
    doc2.add(new BinaryPoint("dim", new byte[6]));
    IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
        w.addDocument(doc2);
    });
    assertEquals("cannot change point numBytes from 4 to 6 for field=\"dim\"", expected.getMessage());
    w.close();
    dir.close();
}