Examples with Reader org.apache.lucene.util.packed.PackedInts.Reader used on opensource projects

Example 1 with Reader

use of org.apache.lucene.util.packed.PackedInts.Reader in project lucene-solr by apache.

the class TestPackedInts method testEndPointer.

public void testEndPointer() throws IOException {
    final Directory dir = newDirectory();
    final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
    final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
    for (int i = 0; i < valueCount; ++i) {
        out.writeLong(0);
    }
    out.close();
    final IndexInput in = dir.openInput("tests.bin", newIOContext(random()));
    for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) {
        for (int bpv = 1; bpv <= 64; ++bpv) {
            for (PackedInts.Format format : PackedInts.Format.values()) {
                if (!format.isSupported(bpv)) {
                    continue;
                }
                final long byteCount = format.byteCount(version, valueCount, bpv);
                String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;
                // test iterator
                in.seek(0L);
                final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1 << 16));
                for (int i = 0; i < valueCount; ++i) {
                    it.next();
                }
                assertEquals(msg, byteCount, in.getFilePointer());
                // test direct reader
                in.seek(0L);
                final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv);
                directReader.get(valueCount - 1);
                assertEquals(msg, byteCount, in.getFilePointer());
                // test reader
                in.seek(0L);
                PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv);
                assertEquals(msg, byteCount, in.getFilePointer());
            }
        }
    }
    in.close();
    dir.close();
}

Example 2 with Reader

use of org.apache.lucene.util.packed.PackedInts.Reader in project lucene-solr by apache.

the class TestPackedInts method testSave.

public void testSave() throws IOException {
    final int valueCount = TestUtil.nextInt(random(), 1, 2048);
    for (int bpv = 1; bpv <= 64; ++bpv) {
        final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
        final RAMDirectory directory = new RAMDirectory();
        List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
        for (PackedInts.Mutable mutable : packedInts) {
            for (int i = 0; i < mutable.size(); ++i) {
                mutable.set(i, random().nextInt(maxValue));
            }
            IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
            mutable.save(out);
            out.close();
            IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
            PackedInts.Reader reader = PackedInts.getReader(in);
            assertEquals(valueCount, reader.size());
            if (mutable instanceof Packed64SingleBlock) {
                // make sure that we used the right format so that the reader has
                // the same performance characteristics as the mutable that has been
                // serialized
                assertTrue(reader instanceof Packed64SingleBlock);
            } else {
                assertFalse(reader instanceof Packed64SingleBlock);
            }
            for (int i = 0; i < valueCount; ++i) {
                assertEquals(mutable.get(i), reader.get(i));
            }
            in.close();
            directory.deleteFile("packed-ints.bin");
        }
        directory.close();
    }
}

Example 3 with Reader

use of org.apache.lucene.util.packed.PackedInts.Reader in project lucene-solr by apache.

the class TestPackedInts method testPackedInts.

public void testPackedInts() throws IOException {
    int num = atLeast(3);
    for (int iter = 0; iter < num; iter++) {
        for (int nbits = 1; nbits <= 64; nbits++) {
            final long maxValue = PackedInts.maxValue(nbits);
            final int valueCount = TestUtil.nextInt(random(), 1, 600);
            final int bufferSize = random().nextBoolean() ? TestUtil.nextInt(random(), 0, 48) : TestUtil.nextInt(random(), 0, 4096);
            final Directory d = newDirectory();
            IndexOutput out = d.createOutput("out.bin", newIOContext(random()));
            final float acceptableOverhead;
            if (iter == 0) {
                // have the first iteration go through exact nbits
                acceptableOverhead = 0.0f;
            } else {
                acceptableOverhead = random().nextFloat();
            }
            PackedInts.Writer w = PackedInts.getWriter(out, valueCount, nbits, acceptableOverhead);
            final long startFp = out.getFilePointer();
            final int actualValueCount = random().nextBoolean() ? valueCount : TestUtil.nextInt(random(), 0, valueCount);
            final long[] values = new long[valueCount];
            for (int i = 0; i < actualValueCount; i++) {
                if (nbits == 64) {
                    values[i] = random().nextLong();
                } else {
                    values[i] = TestUtil.nextLong(random(), 0, maxValue);
                }
                w.add(values[i]);
            }
            w.finish();
            final long fp = out.getFilePointer();
            out.close();
            // ensure that finish() added the (valueCount-actualValueCount) missing values
            final long bytes = w.getFormat().byteCount(PackedInts.VERSION_CURRENT, valueCount, w.bitsPerValue);
            assertEquals(bytes, fp - startFp);
            {
                // test header
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                // header = codec header | bitsPerValue | valueCount | format
                // codec header
                CodecUtil.checkHeader(in, PackedInts.CODEC_NAME, PackedInts.VERSION_START, PackedInts.VERSION_CURRENT);
                assertEquals(w.bitsPerValue, in.readVInt());
                assertEquals(valueCount, in.readVInt());
                assertEquals(w.getFormat().getId(), in.readVInt());
                assertEquals(startFp, in.getFilePointer());
                in.close();
            }
            {
                // test reader
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.Reader r = PackedInts.getReader(in);
                assertEquals(fp, in.getFilePointer());
                for (int i = 0; i < valueCount; i++) {
                    assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i], r.get(i));
                }
                in.close();
                final long expectedBytesUsed = RamUsageTester.sizeOf(r);
                final long computedBytesUsed = r.ramBytesUsed();
                assertEquals(r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed, expectedBytesUsed, computedBytesUsed);
            }
            {
                // test reader iterator next
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.ReaderIterator r = PackedInts.getReaderIterator(in, bufferSize);
                for (int i = 0; i < valueCount; i++) {
                    assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i], r.next());
                    assertEquals(i, r.ord());
                }
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            {
                // test reader iterator bulk next
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.ReaderIterator r = PackedInts.getReaderIterator(in, bufferSize);
                int i = 0;
                while (i < valueCount) {
                    final int count = TestUtil.nextInt(random(), 1, 95);
                    final LongsRef next = r.next(count);
                    for (int k = 0; k < next.length; ++k) {
                        assertEquals("index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + r.getClass().getSimpleName(), values[i + k], next.longs[next.offset + k]);
                    }
                    i += next.length;
                }
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            {
                // test direct reader get
                IndexInput in = d.openInput("out.bin", newIOContext(random()));
                PackedInts.Reader intsEnum = PackedInts.getDirectReader(in);
                for (int i = 0; i < valueCount; i++) {
                    final String msg = "index=" + i + " valueCount=" + valueCount + " nbits=" + nbits + " for " + intsEnum.getClass().getSimpleName();
                    final int index = random().nextInt(valueCount);
                    assertEquals(msg, values[index], intsEnum.get(index));
                }
                intsEnum.get(intsEnum.size() - 1);
                assertEquals(fp, in.getFilePointer());
                in.close();
            }
            d.close();
        }
    }
}

Example 4 with Reader

use of org.apache.lucene.util.packed.PackedInts.Reader in project lucene-solr by apache.

the class TestPackedInts method testSingleValue.

public void testSingleValue() throws Exception {
    for (int bitsPerValue = 1; bitsPerValue <= 64; ++bitsPerValue) {
        Directory dir = newDirectory();
        IndexOutput out = dir.createOutput("out", newIOContext(random()));
        PackedInts.Writer w = PackedInts.getWriter(out, 1, bitsPerValue, PackedInts.DEFAULT);
        long value = 17L & PackedInts.maxValue(bitsPerValue);
        w.add(value);
        w.finish();
        final long end = out.getFilePointer();
        out.close();
        IndexInput in = dir.openInput("out", newIOContext(random()));
        Reader reader = PackedInts.getReader(in);
        String msg = "Impl=" + w.getClass().getSimpleName() + ", bitsPerValue=" + bitsPerValue;
        assertEquals(msg, 1, reader.size());
        assertEquals(msg, value, reader.get(0));
        assertEquals(msg, end, in.getFilePointer());
        in.close();
        dir.close();
    }
}

Example 5 with Reader

use of org.apache.lucene.util.packed.PackedInts.Reader in project lucene-solr by apache.

the class TestPackedInts method testPackedIntsNull.

public void testPackedIntsNull() {
    // must be > 10 for the bulk reads below
    int size = TestUtil.nextInt(random(), 11, 256);
    Reader packedInts = new PackedInts.NullReader(size);
    assertEquals(0, packedInts.get(TestUtil.nextInt(random(), 0, size - 1)));
    long[] arr = new long[size + 10];
    int r;
    Arrays.fill(arr, 1);
    r = packedInts.get(0, arr, 0, size - 1);
    assertEquals(size - 1, r);
    for (r--; r >= 0; r--) {
        assertEquals(0, arr[r]);
    }
    Arrays.fill(arr, 1);
    r = packedInts.get(10, arr, 0, size + 10);
    assertEquals(size - 10, r);
    for (int i = 0; i < size - 10; i++) {
        assertEquals(0, arr[i]);
    }
}