Examples with IndexInput org.apache.lucene.store.IndexInput used on opensource projects

Example 26 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestPagedBytes method testOverflow.

// memory hole
@Ignore
public void testOverflow() throws IOException {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    final int blockBits = TestUtil.nextInt(random(), 14, 28);
    final int blockSize = 1 << blockBits;
    byte[] arr = new byte[TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
    for (int i = 0; i < arr.length; ++i) {
        arr[i] = (byte) i;
    }
    final long numBytes = (1L << 31) + TestUtil.nextInt(random(), 1, blockSize * 3);
    final PagedBytes p = new PagedBytes(blockBits);
    final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
    for (long i = 0; i < numBytes; ) {
        assertEquals(i, out.getFilePointer());
        final int len = (int) Math.min(arr.length, numBytes - i);
        out.writeBytes(arr, len);
        i += len;
    }
    assertEquals(numBytes, out.getFilePointer());
    out.close();
    final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
    p.copy(in, numBytes);
    final PagedBytes.Reader reader = p.freeze(random().nextBoolean());
    for (long offset : new long[] { 0L, Integer.MAX_VALUE, numBytes - 1, TestUtil.nextLong(random(), 1, numBytes - 2) }) {
        BytesRef b = new BytesRef();
        reader.fillSlice(b, offset, 1);
        assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
    }
    in.close();
    dir.close();
}

Example 27 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestPackedInts method testEndPointer.

public void testEndPointer() throws IOException {
    final Directory dir = newDirectory();
    final int valueCount = RandomNumbers.randomIntBetween(random(), 1, 1000);
    final IndexOutput out = dir.createOutput("tests.bin", newIOContext(random()));
    for (int i = 0; i < valueCount; ++i) {
        out.writeLong(0);
    }
    out.close();
    final IndexInput in = dir.openInput("tests.bin", newIOContext(random()));
    for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; ++version) {
        for (int bpv = 1; bpv <= 64; ++bpv) {
            for (PackedInts.Format format : PackedInts.Format.values()) {
                if (!format.isSupported(bpv)) {
                    continue;
                }
                final long byteCount = format.byteCount(version, valueCount, bpv);
                String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;
                // test iterator
                in.seek(0L);
                final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(in, format, version, valueCount, bpv, RandomNumbers.randomIntBetween(random(), 1, 1 << 16));
                for (int i = 0; i < valueCount; ++i) {
                    it.next();
                }
                assertEquals(msg, byteCount, in.getFilePointer());
                // test direct reader
                in.seek(0L);
                final PackedInts.Reader directReader = PackedInts.getDirectReaderNoHeader(in, format, version, valueCount, bpv);
                directReader.get(valueCount - 1);
                assertEquals(msg, byteCount, in.getFilePointer());
                // test reader
                in.seek(0L);
                PackedInts.getReaderNoHeader(in, format, version, valueCount, bpv);
                assertEquals(msg, byteCount, in.getFilePointer());
            }
        }
    }
    in.close();
    dir.close();
}

Example 28 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestPackedInts method testSave.

public void testSave() throws IOException {
    final int valueCount = TestUtil.nextInt(random(), 1, 2048);
    for (int bpv = 1; bpv <= 64; ++bpv) {
        final int maxValue = (int) Math.min(PackedInts.maxValue(31), PackedInts.maxValue(bpv));
        final RAMDirectory directory = new RAMDirectory();
        List<PackedInts.Mutable> packedInts = createPackedInts(valueCount, bpv);
        for (PackedInts.Mutable mutable : packedInts) {
            for (int i = 0; i < mutable.size(); ++i) {
                mutable.set(i, random().nextInt(maxValue));
            }
            IndexOutput out = directory.createOutput("packed-ints.bin", IOContext.DEFAULT);
            mutable.save(out);
            out.close();
            IndexInput in = directory.openInput("packed-ints.bin", IOContext.DEFAULT);
            PackedInts.Reader reader = PackedInts.getReader(in);
            assertEquals(valueCount, reader.size());
            if (mutable instanceof Packed64SingleBlock) {
                // make sure that we used the right format so that the reader has
                // the same performance characteristics as the mutable that has been
                // serialized
                assertTrue(reader instanceof Packed64SingleBlock);
            } else {
                assertFalse(reader instanceof Packed64SingleBlock);
            }
            for (int i = 0; i < valueCount; ++i) {
                assertEquals(mutable.get(i), reader.get(i));
            }
            in.close();
            directory.deleteFile("packed-ints.bin");
        }
        directory.close();
    }
}

Example 29 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class CompletionsTermsReader method suggester.

/**
   * Returns the suggester for a field, if not loaded already, loads
   * the appropriate suggester from CompletionDictionary
   */
public synchronized NRTSuggester suggester() throws IOException {
    if (suggester == null) {
        try (IndexInput dictClone = dictIn.clone()) {
            // let multiple fields load concurrently
            dictClone.seek(offset);
            suggester = NRTSuggester.load(dictClone);
        }
    }
    return suggester;
}

Example 30 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestBKD method verify.

private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
    int numValues = docValues.length;
    if (VERBOSE) {
        System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
    }
    List<Long> toMerge = null;
    List<MergeState.DocMap> docMaps = null;
    int seg = 0;
    BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
    IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
    IndexInput in = null;
    boolean success = false;
    try {
        byte[] scratch = new byte[numBytesPerDim * numDims];
        int lastDocIDBase = 0;
        boolean useMerge = numDims == 1 && numValues >= 10 && random().nextBoolean();
        int valuesInThisSeg;
        if (useMerge) {
            // Sometimes we will call merge with a single segment:
            valuesInThisSeg = TestUtil.nextInt(random(), numValues / 10, numValues);
        } else {
            valuesInThisSeg = 0;
        }
        int segCount = 0;
        for (int ord = 0; ord < numValues; ord++) {
            int docID;
            if (docIDs == null) {
                docID = ord;
            } else {
                docID = docIDs[ord];
            }
            if (VERBOSE) {
                System.out.println("  ord=" + ord + " docID=" + docID + " lastDocIDBase=" + lastDocIDBase);
            }
            for (int dim = 0; dim < numDims; dim++) {
                if (VERBOSE) {
                    System.out.println("    " + dim + " -> " + new BytesRef(docValues[ord][dim]));
                }
                System.arraycopy(docValues[ord][dim], 0, scratch, dim * numBytesPerDim, numBytesPerDim);
            }
            w.add(scratch, docID - lastDocIDBase);
            segCount++;
            if (useMerge && segCount == valuesInThisSeg) {
                if (toMerge == null) {
                    toMerge = new ArrayList<>();
                    docMaps = new ArrayList<>();
                }
                final int curDocIDBase = lastDocIDBase;
                docMaps.add(new MergeState.DocMap() {

                    @Override
                    public int get(int docID) {
                        return curDocIDBase + docID;
                    }
                });
                toMerge.add(w.finish(out));
                valuesInThisSeg = TestUtil.nextInt(random(), numValues / 10, numValues / 2);
                segCount = 0;
                seg++;
                maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
                maxMB = (float) 3.0 + (3 * random().nextDouble());
                w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
                lastDocIDBase = docID;
            }
        }
        long indexFP;
        if (toMerge != null) {
            if (segCount > 0) {
                toMerge.add(w.finish(out));
                final int curDocIDBase = lastDocIDBase;
                docMaps.add(new MergeState.DocMap() {

                    @Override
                    public int get(int docID) {
                        return curDocIDBase + docID;
                    }
                });
            }
            out.close();
            in = dir.openInput("bkd", IOContext.DEFAULT);
            seg++;
            w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
            List<BKDReader> readers = new ArrayList<>();
            for (long fp : toMerge) {
                in.seek(fp);
                readers.add(new BKDReader(in));
            }
            out = dir.createOutput("bkd2", IOContext.DEFAULT);
            indexFP = w.merge(out, docMaps, readers);
            out.close();
            in.close();
            in = dir.openInput("bkd2", IOContext.DEFAULT);
        } else {
            indexFP = w.finish(out);
            out.close();
            in = dir.openInput("bkd", IOContext.DEFAULT);
        }
        in.seek(indexFP);
        BKDReader r = new BKDReader(in);
        int iters = atLeast(100);
        for (int iter = 0; iter < iters; iter++) {
            if (VERBOSE) {
                System.out.println("\nTEST: iter=" + iter);
            }
            // Random N dims rect query:
            byte[][] queryMin = new byte[numDims][];
            byte[][] queryMax = new byte[numDims][];
            for (int dim = 0; dim < numDims; dim++) {
                queryMin[dim] = new byte[numBytesPerDim];
                random().nextBytes(queryMin[dim]);
                queryMax[dim] = new byte[numBytesPerDim];
                random().nextBytes(queryMax[dim]);
                if (StringHelper.compare(numBytesPerDim, queryMin[dim], 0, queryMax[dim], 0) > 0) {
                    byte[] x = queryMin[dim];
                    queryMin[dim] = queryMax[dim];
                    queryMax[dim] = x;
                }
            }
            final BitSet hits = new BitSet();
            r.intersect(new IntersectVisitor() {

                @Override
                public void visit(int docID) {
                    hits.set(docID);
                //System.out.println("visit docID=" + docID);
                }

                @Override
                public void visit(int docID, byte[] packedValue) {
                    //System.out.println("visit check docID=" + docID);
                    for (int dim = 0; dim < numDims; dim++) {
                        if (StringHelper.compare(numBytesPerDim, packedValue, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, packedValue, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            //System.out.println("  no");
                            return;
                        }
                    }
                    //System.out.println("  yes");
                    hits.set(docID);
                }

                @Override
                public Relation compare(byte[] minPacked, byte[] maxPacked) {
                    boolean crosses = false;
                    for (int dim = 0; dim < numDims; dim++) {
                        if (StringHelper.compare(numBytesPerDim, maxPacked, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, minPacked, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            return Relation.CELL_OUTSIDE_QUERY;
                        } else if (StringHelper.compare(numBytesPerDim, minPacked, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, maxPacked, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            crosses = true;
                        }
                    }
                    if (crosses) {
                        return Relation.CELL_CROSSES_QUERY;
                    } else {
                        return Relation.CELL_INSIDE_QUERY;
                    }
                }
            });
            BitSet expected = new BitSet();
            for (int ord = 0; ord < numValues; ord++) {
                boolean matches = true;
                for (int dim = 0; dim < numDims; dim++) {
                    byte[] x = docValues[ord][dim];
                    if (StringHelper.compare(numBytesPerDim, x, 0, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, x, 0, queryMax[dim], 0) > 0) {
                        matches = false;
                        break;
                    }
                }
                if (matches) {
                    int docID;
                    if (docIDs == null) {
                        docID = ord;
                    } else {
                        docID = docIDs[ord];
                    }
                    expected.set(docID);
                }
            }
            int limit = Math.max(expected.length(), hits.length());
            for (int docID = 0; docID < limit; docID++) {
                assertEquals("docID=" + docID, expected.get(docID), hits.get(docID));
            }
        }
        in.close();
        dir.deleteFile("bkd");
        if (toMerge != null) {
            dir.deleteFile("bkd2");
        }
        success = true;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(w, in, out);
            IOUtils.deleteFilesIgnoringExceptions(dir, "bkd", "bkd2");
        }
    }
}