Examples with LongsRef org.apache.lucene.util.LongsRef used on opensource projects

Example 6 with LongsRef

use of org.apache.lucene.util.LongsRef in project lucene-solr by apache.

the class TestPackedInts method testPackedInts.

public void testPackedInts() throws IOException {
    int num = atLeast(3);
    for (int iter = 0; iter < num; iter++) {
        for (int nbits = 1; nbits <= 64; nbits++) {
            final long maxValue = PackedInts.maxValue(nbits);
            final int valueCount = TestUtil.nextInt(random(), 1, 600);
            final int bufferSize = random().nextBoolean() ? TestUtil.nextInt(random(), 0, 48) : TestUtil.nextInt(random(), 0, 4096);
            final Directory d = newDirectory();
            IndexOutput out = d.createOutput("out.bin", newIOContext(random()));
            final float acceptableOverhead;
            if (iter == 0) {
                // have the first iteration go through exact nbits
                acceptableOverhead = 0.0f;
            } else {
                acceptableOverhead = random().nextFloat();
            }
            PackedInts.Writer w = PackedInts.getWriter(out, valueCount, nbits, acceptableOverhead);
            final long startFp = out.getFilePointer();
            final int actualValueCount = random().nextBoolean() ? valueCount : TestUtil.nextInt(random(), 0, valueCount);
            final long[] values = new long[valueCount];
            for (int i = 0; i < actualValueCount; i++) {
                if (nbits == 64) {
                    values[i] = random().nextLong();
                } else {
                    values[i] = TestUtil.nextLong(random(), 0, maxValue);
                }
                w.add(values[i]);
            }
            w.finish();
            final long fp = out.getFilePointer();
            out.close();
            // ensure that finish() added the (valueCount-actualValueCount) missing values
            final long bytes = w.getFormat().byteCount(PackedInts.VERSION_CURRENT, valueCount, w.bitsPerValue);
            assertEquals(bytes, fp - startFp);
            {
                // test header
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                // header = codec header | bitsPerValue | valueCount | format
                // codec header
                CodecUtil.checkHeader(in, PackedInts.CODEC_NAME, PackedInts.VERSION_START, PackedInts.VERSION_CURRENT);
                assertEquals(w.bitsPerValue, in.readVInt());
                assertEquals(valueCount, in.readVInt());
                assertEquals(w.getFormat().getId(), in.readVInt());
                assertEquals(startFp, in.getFilePointer());
                in.close();
            }
            {
                // test reader
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.Reader r = PackedInts.getReader(in);
                assertEquals(fp, in.getFilePointer());
                for (int i = 0; i < valueCount; i++) {
                    assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i], r.get(i));
                }
                in.close();
                final long expectedBytesUsed = RamUsageTester.sizeOf(r);
                final long computedBytesUsed = r.ramBytesUsed();
                assertEquals(r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed, expectedBytesUsed, computedBytesUsed);
            }
            {
                // test reader iterator next
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.ReaderIterator r = PackedInts.getReaderIterator(in, bufferSize);
                for (int i = 0; i < valueCount; i++) {
                    assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i], r.next());
                    assertEquals(i, r.ord());
                }
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            {
                // test reader iterator bulk next
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.ReaderIterator r = PackedInts.getReaderIterator(in, bufferSize);
                int i = 0;
                while (i < valueCount) {
                    final int count = TestUtil.nextInt(random(), 1, 95);
                    final LongsRef next = r.next(count);
                    for (int k = 0; k < next.length; ++k) {
                        assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i + k], next.longs[next.offset + k]);
                    }
                    i += next.length;
                }
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            {
                // test direct reader get
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.Reader intsEnum = PackedInts.getDirectReader(in);
                for (int i = 0; i < valueCount; i++) {
                    final String msg = "index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + intsEnum.getClass().getSimpleName();
                    final int index = random().nextInt(valueCount);
                    assertEquals(msg, values[index], intsEnum.get(index));
                }
                intsEnum.get(intsEnum.size() - 1);
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            d.close();
        }
    }
}

Example 7 with LongsRef

use of org.apache.lucene.util.LongsRef in project lucene-solr by apache.

the class TestPackedInts method testBlockPackedReaderWriter.

public void testBlockPackedReaderWriter() throws IOException {
    final int iters = atLeast(2);
    for (int iter = 0; iter < iters; ++iter) {
        final int blockSize = 1 << TestUtil.nextInt(random(), 6, 18);
        final int valueCount = random().nextInt(1 << 18);
        final long[] values = new long[valueCount];
        long minValue = 0;
        int bpv = 0;
        for (int i = 0; i < valueCount; ++i) {
            if (i % blockSize == 0) {
                minValue = rarely() ? random().nextInt(256) : rarely() ? -5 : random().nextLong();
                bpv = random().nextInt(65);
            }
            if (bpv == 0) {
                values[i] = minValue;
            } else if (bpv == 64) {
                values[i] = random().nextLong();
            } else {
                values[i] = minValue + TestUtil.nextLong(random(), 0, (1L << bpv) - 1);
            }
        }
        final Directory dir = newDirectory();
        final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
        final BlockPackedWriter writer = new BlockPackedWriter(out, blockSize);
        for (int i = 0; i < valueCount; ++i) {
            assertEquals(i, writer.ord());
            writer.add(values[i]);
        }
        assertEquals(valueCount, writer.ord());
        writer.finish();
        assertEquals(valueCount, writer.ord());
        final long fp = out.getFilePointer();
        out.close();
        IndexInput in1 = dir.openInput("out.bin", IOContext.DEFAULT);
        byte[] buf = new byte[(int) fp];
        in1.readBytes(buf, 0, (int) fp);
        in1.seek(0L);
        ByteArrayDataInput in2 = new ByteArrayDataInput(buf);
        final DataInput in = random().nextBoolean() ? in1 : in2;
        final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
        for (int i = 0; i < valueCount; ) {
            if (random().nextBoolean()) {
                assertEquals("" + i, values[i], it.next());
                ++i;
            } else {
                final LongsRef nextValues = it.next(TestUtil.nextInt(random(), 1, 1024));
                for (int j = 0; j < nextValues.length; ++j) {
                    assertEquals("" + (i + j), values[i + j], nextValues.longs[nextValues.offset + j]);
                }
                i += nextValues.length;
            }
            assertEquals(i, it.ord());
        }
        assertEquals(fp, in instanceof ByteArrayDataInput ? ((ByteArrayDataInput) in).getPosition() : ((IndexInput) in).getFilePointer());
        expectThrows(IOException.class, () -> {
            it.next();
        });
        if (in instanceof ByteArrayDataInput) {
            ((ByteArrayDataInput) in).setPosition(0);
        } else {
            ((IndexInput) in).seek(0L);
        }
        final BlockPackedReaderIterator it2 = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
        int i = 0;
        while (true) {
            final int skip = TestUtil.nextInt(random(), 0, valueCount - i);
            it2.skip(skip);
            i += skip;
            assertEquals(i, it2.ord());
            if (i == valueCount) {
                break;
            } else {
                assertEquals(values[i], it2.next());
                ++i;
            }
        }
        assertEquals(fp, in instanceof ByteArrayDataInput ? ((ByteArrayDataInput) in).getPosition() : ((IndexInput) in).getFilePointer());
        expectThrows(IOException.class, () -> {
            it2.skip(1);
        });
        in1.seek(0L);
        final BlockPackedReader reader = new BlockPackedReader(in1, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
        assertEquals(in1.getFilePointer(), in1.length());
        for (i = 0; i < valueCount; ++i) {
            assertEquals("i=" + i, values[i], reader.get(i));
        }
        in1.close();
        dir.close();
    }
}