Examples with IndexOutput org.apache.lucene.store.IndexOutput used on opensource projects

Example 36 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestBKD method testRandomIntsNDims.

public void testRandomIntsNDims() throws Exception {
    int numDocs = atLeast(1000);
    try (Directory dir = getDirectory(numDocs)) {
        int numDims = TestUtil.nextInt(random(), 1, 5);
        int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
        float maxMB = (float) 3.0 + (3 * random().nextFloat());
        BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, numDocs, true);
        if (VERBOSE) {
            System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs);
        }
        int[][] docs = new int[numDocs][];
        byte[] scratch = new byte[4 * numDims];
        int[] minValue = new int[numDims];
        int[] maxValue = new int[numDims];
        Arrays.fill(minValue, Integer.MAX_VALUE);
        Arrays.fill(maxValue, Integer.MIN_VALUE);
        for (int docID = 0; docID < numDocs; docID++) {
            int[] values = new int[numDims];
            if (VERBOSE) {
                System.out.println("  docID=" + docID);
            }
            for (int dim = 0; dim < numDims; dim++) {
                values[dim] = random().nextInt();
                if (values[dim] < minValue[dim]) {
                    minValue[dim] = values[dim];
                }
                if (values[dim] > maxValue[dim]) {
                    maxValue[dim] = values[dim];
                }
                NumericUtils.intToSortableBytes(values[dim], scratch, dim * Integer.BYTES);
                if (VERBOSE) {
                    System.out.println("    " + dim + " -> " + values[dim]);
                }
            }
            docs[docID] = values;
            w.add(scratch, docID);
        }
        long indexFP;
        try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
            indexFP = w.finish(out);
        }
        try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
            in.seek(indexFP);
            BKDReader r = new BKDReader(in);
            byte[] minPackedValue = r.getMinPackedValue();
            byte[] maxPackedValue = r.getMaxPackedValue();
            for (int dim = 0; dim < numDims; dim++) {
                assertEquals(minValue[dim], NumericUtils.sortableBytesToInt(minPackedValue, dim * Integer.BYTES));
                assertEquals(maxValue[dim], NumericUtils.sortableBytesToInt(maxPackedValue, dim * Integer.BYTES));
            }
            int iters = atLeast(100);
            for (int iter = 0; iter < iters; iter++) {
                if (VERBOSE) {
                    System.out.println("\nTEST: iter=" + iter);
                }
                // Random N dims rect query:
                int[] queryMin = new int[numDims];
                int[] queryMax = new int[numDims];
                for (int dim = 0; dim < numDims; dim++) {
                    queryMin[dim] = random().nextInt();
                    queryMax[dim] = random().nextInt();
                    if (queryMin[dim] > queryMax[dim]) {
                        int x = queryMin[dim];
                        queryMin[dim] = queryMax[dim];
                        queryMax[dim] = x;
                    }
                }
                final BitSet hits = new BitSet();
                r.intersect(new IntersectVisitor() {

                    @Override
                    public void visit(int docID) {
                        hits.set(docID);
                    //System.out.println("visit docID=" + docID);
                    }

                    @Override
                    public void visit(int docID, byte[] packedValue) {
                        //System.out.println("visit check docID=" + docID);
                        for (int dim = 0; dim < numDims; dim++) {
                            int x = NumericUtils.sortableBytesToInt(packedValue, dim * Integer.BYTES);
                            if (x < queryMin[dim] || x > queryMax[dim]) {
                                //System.out.println("  no");
                                return;
                            }
                        }
                        //System.out.println("  yes");
                        hits.set(docID);
                    }

                    @Override
                    public Relation compare(byte[] minPacked, byte[] maxPacked) {
                        boolean crosses = false;
                        for (int dim = 0; dim < numDims; dim++) {
                            int min = NumericUtils.sortableBytesToInt(minPacked, dim * Integer.BYTES);
                            int max = NumericUtils.sortableBytesToInt(maxPacked, dim * Integer.BYTES);
                            assert max >= min;
                            if (max < queryMin[dim] || min > queryMax[dim]) {
                                return Relation.CELL_OUTSIDE_QUERY;
                            } else if (min < queryMin[dim] || max > queryMax[dim]) {
                                crosses = true;
                            }
                        }
                        if (crosses) {
                            return Relation.CELL_CROSSES_QUERY;
                        } else {
                            return Relation.CELL_INSIDE_QUERY;
                        }
                    }
                });
                for (int docID = 0; docID < numDocs; docID++) {
                    int[] docValues = docs[docID];
                    boolean expected = true;
                    for (int dim = 0; dim < numDims; dim++) {
                        int x = docValues[dim];
                        if (x < queryMin[dim] || x > queryMax[dim]) {
                            expected = false;
                            break;
                        }
                    }
                    boolean actual = hits.get(docID);
                    assertEquals("docID=" + docID, expected, actual);
                }
            }
        }
    }
}

Example 37 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestBKD method testTieBreakOrder.

public void testTieBreakOrder() throws Exception {
    try (Directory dir = newDirectory()) {
        int numDocs = 10000;
        BKDWriter w = new BKDWriter(numDocs + 1, dir, "tmp", 1, Integer.BYTES, 2, 0.01f, numDocs, true);
        for (int i = 0; i < numDocs; i++) {
            w.add(new byte[Integer.BYTES], i);
        }
        IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
        long fp = w.finish(out);
        out.close();
        IndexInput in = dir.openInput("bkd", IOContext.DEFAULT);
        in.seek(fp);
        BKDReader r = new BKDReader(in);
        r.intersect(new IntersectVisitor() {

            int lastDocID = -1;

            @Override
            public void visit(int docID) {
                assertTrue("lastDocID=" + lastDocID + " docID=" + docID, docID > lastDocID);
                lastDocID = docID;
            }

            @Override
            public void visit(int docID, byte[] packedValue) {
                visit(docID);
            }

            @Override
            public Relation compare(byte[] minPacked, byte[] maxPacked) {
                return Relation.CELL_CROSSES_QUERY;
            }
        });
        in.close();
    }
}

Example 38 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestBKD method testBitFlippedOnPartition2.

/** Make sure corruption on a recursed partition is caught, when BKDWriter does get angry */
public void testBitFlippedOnPartition2() throws Exception {
    // Generate fixed data set:
    int numDocs = atLeast(10000);
    int numBytesPerDim = 4;
    int numDims = 3;
    byte[][][] docValues = new byte[numDocs][][];
    byte counter = 0;
    for (int docID = 0; docID < numDocs; docID++) {
        byte[][] values = new byte[numDims][];
        for (int dim = 0; dim < numDims; dim++) {
            values[dim] = new byte[numBytesPerDim];
            for (int i = 0; i < values[dim].length; i++) {
                values[dim][i] = counter;
                counter++;
            }
        }
        docValues[docID] = values;
    }
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                //System.out.println("prefix=" + prefix + " suffix=" + suffix);
                if (corrupted == false && suffix.equals("bkd_left1")) {
                    //System.out.println("now corrupt byte=" + x + " prefix=" + prefix + " suffix=" + suffix);
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 22072, out);
                } else {
                    return out;
                }
            }
        };
        Throwable t = expectThrows(CorruptIndexException.class, () -> {
            verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
        });
        assertCorruptionDetected(t);
    }
}

Example 39 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestDocIdsWriter method test.

private void test(Directory dir, int[] ints) throws Exception {
    final long len;
    try (IndexOutput out = dir.createOutput("tmp", IOContext.DEFAULT)) {
        DocIdsWriter.writeDocIds(ints, 0, ints.length, out);
        len = out.getFilePointer();
        if (random().nextBoolean()) {
            // garbage
            out.writeLong(0);
        }
    }
    try (IndexInput in = dir.openInput("tmp", IOContext.READONCE)) {
        int[] read = new int[ints.length];
        DocIdsWriter.readInts(in, ints.length, read);
        assertArrayEquals(ints, read);
        assertEquals(len, in.getFilePointer());
    }
    try (IndexInput in = dir.openInput("tmp", IOContext.READONCE)) {
        int[] read = new int[ints.length];
        DocIdsWriter.readInts(in, ints.length, new IntersectVisitor() {

            int i = 0;

            @Override
            public void visit(int docID) throws IOException {
                read[i++] = docID;
            }

            @Override
            public void visit(int docID, byte[] packedValue) throws IOException {
                throw new UnsupportedOperationException();
            }

            @Override
            public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
                throw new UnsupportedOperationException();
            }
        });
        assertArrayEquals(ints, read);
        assertEquals(len, in.getFilePointer());
    }
    dir.deleteFile("tmp");
}

Example 40 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestDirectMonotonic method testEmpty.

public void testEmpty() throws IOException {
    Directory dir = newDirectory();
    final int blockShift = TestUtil.nextInt(random(), DirectMonotonicWriter.MIN_BLOCK_SHIFT, DirectMonotonicWriter.MAX_BLOCK_SHIFT);
    final long dataLength;
    try (IndexOutput metaOut = dir.createOutput("meta", IOContext.DEFAULT);
        IndexOutput dataOut = dir.createOutput("data", IOContext.DEFAULT)) {
        DirectMonotonicWriter w = DirectMonotonicWriter.getInstance(metaOut, dataOut, 0, blockShift);
        w.finish();
        dataLength = dataOut.getFilePointer();
    }
    try (IndexInput metaIn = dir.openInput("meta", IOContext.READONCE);
        IndexInput dataIn = dir.openInput("data", IOContext.DEFAULT)) {
        DirectMonotonicReader.Meta meta = DirectMonotonicReader.loadMeta(metaIn, 0, blockShift);
        DirectMonotonicReader.getInstance(meta, dataIn.randomAccessSlice(0, dataLength));
    // no exception
    }
    dir.close();
}