Search in sources :

Example 96 with BitSet

use of java.util.BitSet in project poi by apache.

the class CellNumberFormatter method formatValue.

/** {@inheritDoc} */
public void formatValue(StringBuffer toAppendTo, Object valueObject) {
    double value = ((Number) valueObject).doubleValue();
    value *= scale;
    // For negative numbers:
    // - If the cell format has a negative number format, this method
    // is called with a positive value and the number format has
    // the negative formatting required, e.g. minus sign or brackets.
    // - If the cell format does not have a negative number format,
    // this method is called with a negative value and the number is
    // formatted with a minus sign at the start.
    boolean negative = value < 0;
    if (negative)
        value = -value;
    // Split out the fractional part if we need to print a fraction
    double fractional = 0;
    if (slash != null) {
        if (improperFraction) {
            fractional = value;
            value = 0;
        } else {
            fractional = value % 1.0;
            //noinspection SillyAssignment
            value = (long) value;
        }
    }
    Set<CellNumberStringMod> mods = new TreeSet<CellNumberStringMod>();
    StringBuffer output = new StringBuffer(desc);
    if (exponent != null) {
        writeScientific(value, output, mods);
    } else if (improperFraction) {
        writeFraction(value, null, fractional, output, mods);
    } else {
        StringBuffer result = new StringBuffer();
        Formatter f = new Formatter(result, LocaleUtil.getUserLocale());
        try {
            f.format(LocaleUtil.getUserLocale(), printfFmt, value);
        } finally {
            f.close();
        }
        if (numerator == null) {
            writeFractional(result, output);
            writeInteger(result, output, integerSpecials, mods, integerCommas);
        } else {
            writeFraction(value, result, fractional, output, mods);
        }
    }
    // Now strip out any remaining '#'s and add any pending text ...
    Iterator<CellNumberStringMod> changes = mods.iterator();
    CellNumberStringMod nextChange = (changes.hasNext() ? changes.next() : null);
    // records chars already deleted
    BitSet deletedChars = new BitSet();
    int adjust = 0;
    for (Special s : specials) {
        int adjustedPos = s.pos + adjust;
        if (!deletedChars.get(s.pos) && output.charAt(adjustedPos) == '#') {
            output.deleteCharAt(adjustedPos);
            adjust--;
            deletedChars.set(s.pos);
        }
        while (nextChange != null && s == nextChange.getSpecial()) {
            int lenBefore = output.length();
            int modPos = s.pos + adjust;
            switch(nextChange.getOp()) {
                case CellNumberStringMod.AFTER:
                    // ignore adding a comma after a deleted char (which was a '#')
                    if (nextChange.getToAdd().equals(",") && deletedChars.get(s.pos)) {
                        break;
                    }
                    output.insert(modPos + 1, nextChange.getToAdd());
                    break;
                case CellNumberStringMod.BEFORE:
                    output.insert(modPos, nextChange.getToAdd());
                    break;
                case CellNumberStringMod.REPLACE:
                    // delete starting pos in original coordinates
                    int delPos = s.pos;
                    if (!nextChange.isStartInclusive()) {
                        delPos++;
                        modPos++;
                    }
                    // Skip over anything already deleted
                    while (deletedChars.get(delPos)) {
                        delPos++;
                        modPos++;
                    }
                    // delete end point in original
                    int delEndPos = nextChange.getEnd().pos;
                    if (nextChange.isEndInclusive()) {
                        delEndPos++;
                    }
                    // delete end point in current
                    int modEndPos = delEndPos + adjust;
                    if (modPos < modEndPos) {
                        if ("".equals(nextChange.getToAdd())) {
                            output.delete(modPos, modEndPos);
                        } else {
                            char fillCh = nextChange.getToAdd().charAt(0);
                            for (int i = modPos; i < modEndPos; i++) {
                                output.setCharAt(i, fillCh);
                            }
                        }
                        deletedChars.set(delPos, delEndPos);
                    }
                    break;
                default:
                    throw new IllegalStateException("Unknown op: " + nextChange.getOp());
            }
            adjust += output.length() - lenBefore;
            nextChange = (changes.hasNext()) ? changes.next() : null;
        }
    }
    // Finally, add it to the string
    if (negative) {
        toAppendTo.append('-');
    }
    toAppendTo.append(output);
}
Also used : Formatter(java.util.Formatter) BitSet(java.util.BitSet) TreeSet(java.util.TreeSet)

Example 97 with BitSet

use of java.util.BitSet in project lucene-solr by apache.

the class BasePointsFormatTestCase method testBasic.

public void testBasic() throws Exception {
    Directory dir = getDirectory(20);
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setMergePolicy(newLogMergePolicy());
    IndexWriter w = new IndexWriter(dir, iwc);
    byte[] point = new byte[4];
    for (int i = 0; i < 20; i++) {
        Document doc = new Document();
        NumericUtils.intToSortableBytes(i, point, 0);
        doc.add(new BinaryPoint("dim", point));
        w.addDocument(doc);
    }
    w.forceMerge(1);
    w.close();
    DirectoryReader r = DirectoryReader.open(dir);
    LeafReader sub = getOnlyLeafReader(r);
    PointValues values = sub.getPointValues("dim");
    // Simple test: make sure intersect can visit every doc:
    BitSet seen = new BitSet();
    values.intersect(new IntersectVisitor() {

        @Override
        public Relation compare(byte[] minPacked, byte[] maxPacked) {
            return Relation.CELL_CROSSES_QUERY;
        }

        public void visit(int docID) {
            throw new IllegalStateException();
        }

        public void visit(int docID, byte[] packedValue) {
            seen.set(docID);
            assertEquals(docID, NumericUtils.sortableBytesToInt(packedValue, 0));
        }
    });
    assertEquals(20, seen.cardinality());
    IOUtils.close(r, dir);
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) BinaryPoint(org.apache.lucene.document.BinaryPoint) BitSet(java.util.BitSet) Document(org.apache.lucene.document.Document) BinaryPoint(org.apache.lucene.document.BinaryPoint) IntPoint(org.apache.lucene.document.IntPoint) Relation(org.apache.lucene.index.PointValues.Relation) Directory(org.apache.lucene.store.Directory)

Example 98 with BitSet

use of java.util.BitSet in project lucene-solr by apache.

the class BasePointsFormatTestCase method testMerge.

public void testMerge() throws Exception {
    Directory dir = getDirectory(20);
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setMergePolicy(newLogMergePolicy());
    IndexWriter w = new IndexWriter(dir, iwc);
    byte[] point = new byte[4];
    for (int i = 0; i < 20; i++) {
        Document doc = new Document();
        NumericUtils.intToSortableBytes(i, point, 0);
        doc.add(new BinaryPoint("dim", point));
        w.addDocument(doc);
        if (i == 10) {
            w.commit();
        }
    }
    w.forceMerge(1);
    w.close();
    DirectoryReader r = DirectoryReader.open(dir);
    LeafReader sub = getOnlyLeafReader(r);
    PointValues values = sub.getPointValues("dim");
    // Simple test: make sure intersect can visit every doc:
    BitSet seen = new BitSet();
    values.intersect(new IntersectVisitor() {

        @Override
        public Relation compare(byte[] minPacked, byte[] maxPacked) {
            return Relation.CELL_CROSSES_QUERY;
        }

        public void visit(int docID) {
            throw new IllegalStateException();
        }

        public void visit(int docID, byte[] packedValue) {
            seen.set(docID);
            assertEquals(docID, NumericUtils.sortableBytesToInt(packedValue, 0));
        }
    });
    assertEquals(20, seen.cardinality());
    IOUtils.close(r, dir);
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) BinaryPoint(org.apache.lucene.document.BinaryPoint) BitSet(java.util.BitSet) Document(org.apache.lucene.document.Document) BinaryPoint(org.apache.lucene.document.BinaryPoint) IntPoint(org.apache.lucene.document.IntPoint) Relation(org.apache.lucene.index.PointValues.Relation) Directory(org.apache.lucene.store.Directory)

Example 99 with BitSet

use of java.util.BitSet in project lucene-solr by apache.

the class TestBKD method verify.

private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDims, int numBytesPerDim, int maxPointsInLeafNode, double maxMB) throws Exception {
    int numValues = docValues.length;
    if (VERBOSE) {
        System.out.println("TEST: numValues=" + numValues + " numDims=" + numDims + " numBytesPerDim=" + numBytesPerDim + " maxPointsInLeafNode=" + maxPointsInLeafNode + " maxMB=" + maxMB);
    }
    List<Long> toMerge = null;
    List<MergeState.DocMap> docMaps = null;
    int seg = 0;
    BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
    IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT);
    IndexInput in = null;
    boolean success = false;
    try {
        byte[] scratch = new byte[numBytesPerDim * numDims];
        int lastDocIDBase = 0;
        boolean useMerge = numDims == 1 && numValues >= 10 && random().nextBoolean();
        int valuesInThisSeg;
        if (useMerge) {
            // Sometimes we will call merge with a single segment:
            valuesInThisSeg = TestUtil.nextInt(random(), numValues / 10, numValues);
        } else {
            valuesInThisSeg = 0;
        }
        int segCount = 0;
        for (int ord = 0; ord < numValues; ord++) {
            int docID;
            if (docIDs == null) {
                docID = ord;
            } else {
                docID = docIDs[ord];
            }
            if (VERBOSE) {
                System.out.println("  ord=" + ord + " docID=" + docID + " lastDocIDBase=" + lastDocIDBase);
            }
            for (int dim = 0; dim < numDims; dim++) {
                if (VERBOSE) {
                    System.out.println("    " + dim + " -> " + new BytesRef(docValues[ord][dim]));
                }
                System.arraycopy(docValues[ord][dim], 0, scratch, dim * numBytesPerDim, numBytesPerDim);
            }
            w.add(scratch, docID - lastDocIDBase);
            segCount++;
            if (useMerge && segCount == valuesInThisSeg) {
                if (toMerge == null) {
                    toMerge = new ArrayList<>();
                    docMaps = new ArrayList<>();
                }
                final int curDocIDBase = lastDocIDBase;
                docMaps.add(new MergeState.DocMap() {

                    @Override
                    public int get(int docID) {
                        return curDocIDBase + docID;
                    }
                });
                toMerge.add(w.finish(out));
                valuesInThisSeg = TestUtil.nextInt(random(), numValues / 10, numValues / 2);
                segCount = 0;
                seg++;
                maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000);
                maxMB = (float) 3.0 + (3 * random().nextDouble());
                w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
                lastDocIDBase = docID;
            }
        }
        long indexFP;
        if (toMerge != null) {
            if (segCount > 0) {
                toMerge.add(w.finish(out));
                final int curDocIDBase = lastDocIDBase;
                docMaps.add(new MergeState.DocMap() {

                    @Override
                    public int get(int docID) {
                        return curDocIDBase + docID;
                    }
                });
            }
            out.close();
            in = dir.openInput("bkd", IOContext.DEFAULT);
            seg++;
            w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false);
            List<BKDReader> readers = new ArrayList<>();
            for (long fp : toMerge) {
                in.seek(fp);
                readers.add(new BKDReader(in));
            }
            out = dir.createOutput("bkd2", IOContext.DEFAULT);
            indexFP = w.merge(out, docMaps, readers);
            out.close();
            in.close();
            in = dir.openInput("bkd2", IOContext.DEFAULT);
        } else {
            indexFP = w.finish(out);
            out.close();
            in = dir.openInput("bkd", IOContext.DEFAULT);
        }
        in.seek(indexFP);
        BKDReader r = new BKDReader(in);
        int iters = atLeast(100);
        for (int iter = 0; iter < iters; iter++) {
            if (VERBOSE) {
                System.out.println("\nTEST: iter=" + iter);
            }
            // Random N dims rect query:
            byte[][] queryMin = new byte[numDims][];
            byte[][] queryMax = new byte[numDims][];
            for (int dim = 0; dim < numDims; dim++) {
                queryMin[dim] = new byte[numBytesPerDim];
                random().nextBytes(queryMin[dim]);
                queryMax[dim] = new byte[numBytesPerDim];
                random().nextBytes(queryMax[dim]);
                if (StringHelper.compare(numBytesPerDim, queryMin[dim], 0, queryMax[dim], 0) > 0) {
                    byte[] x = queryMin[dim];
                    queryMin[dim] = queryMax[dim];
                    queryMax[dim] = x;
                }
            }
            final BitSet hits = new BitSet();
            r.intersect(new IntersectVisitor() {

                @Override
                public void visit(int docID) {
                    hits.set(docID);
                //System.out.println("visit docID=" + docID);
                }

                @Override
                public void visit(int docID, byte[] packedValue) {
                    //System.out.println("visit check docID=" + docID);
                    for (int dim = 0; dim < numDims; dim++) {
                        if (StringHelper.compare(numBytesPerDim, packedValue, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, packedValue, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            //System.out.println("  no");
                            return;
                        }
                    }
                    //System.out.println("  yes");
                    hits.set(docID);
                }

                @Override
                public Relation compare(byte[] minPacked, byte[] maxPacked) {
                    boolean crosses = false;
                    for (int dim = 0; dim < numDims; dim++) {
                        if (StringHelper.compare(numBytesPerDim, maxPacked, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, minPacked, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            return Relation.CELL_OUTSIDE_QUERY;
                        } else if (StringHelper.compare(numBytesPerDim, minPacked, dim * numBytesPerDim, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, maxPacked, dim * numBytesPerDim, queryMax[dim], 0) > 0) {
                            crosses = true;
                        }
                    }
                    if (crosses) {
                        return Relation.CELL_CROSSES_QUERY;
                    } else {
                        return Relation.CELL_INSIDE_QUERY;
                    }
                }
            });
            BitSet expected = new BitSet();
            for (int ord = 0; ord < numValues; ord++) {
                boolean matches = true;
                for (int dim = 0; dim < numDims; dim++) {
                    byte[] x = docValues[ord][dim];
                    if (StringHelper.compare(numBytesPerDim, x, 0, queryMin[dim], 0) < 0 || StringHelper.compare(numBytesPerDim, x, 0, queryMax[dim], 0) > 0) {
                        matches = false;
                        break;
                    }
                }
                if (matches) {
                    int docID;
                    if (docIDs == null) {
                        docID = ord;
                    } else {
                        docID = docIDs[ord];
                    }
                    expected.set(docID);
                }
            }
            int limit = Math.max(expected.length(), hits.length());
            for (int docID = 0; docID < limit; docID++) {
                assertEquals("docID=" + docID, expected.get(docID), hits.get(docID));
            }
        }
        in.close();
        dir.deleteFile("bkd");
        if (toMerge != null) {
            dir.deleteFile("bkd2");
        }
        success = true;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(w, in, out);
            IOUtils.deleteFilesIgnoringExceptions(dir, "bkd", "bkd2");
        }
    }
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) MergeState(org.apache.lucene.index.MergeState) ArrayList(java.util.ArrayList) BitSet(java.util.BitSet) CorruptingIndexOutput(org.apache.lucene.store.CorruptingIndexOutput) IndexOutput(org.apache.lucene.store.IndexOutput) Relation(org.apache.lucene.index.PointValues.Relation) IndexInput(org.apache.lucene.store.IndexInput) BytesRef(org.apache.lucene.util.BytesRef)

Example 100 with BitSet

use of java.util.BitSet in project lucene-solr by apache.

the class TestBKD method testBigIntNDims.

// Tests on N-dimensional points where each dimension is a BigInteger
public void testBigIntNDims() throws Exception {
    int numDocs = atLeast(1000);
    try (Directory dir = getDirectory(numDocs)) {
        int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
        int numDims = TestUtil.nextInt(random(), 1, 5);
        int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100);
        float maxMB = (float) 3.0 + (3 * random().nextFloat());
        BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true);
        BigInteger[][] docs = new BigInteger[numDocs][];
        byte[] scratch = new byte[numBytesPerDim * numDims];
        for (int docID = 0; docID < numDocs; docID++) {
            BigInteger[] values = new BigInteger[numDims];
            if (VERBOSE) {
                System.out.println("  docID=" + docID);
            }
            for (int dim = 0; dim < numDims; dim++) {
                values[dim] = randomBigInt(numBytesPerDim);
                NumericUtils.bigIntToSortableBytes(values[dim], numBytesPerDim, scratch, dim * numBytesPerDim);
                if (VERBOSE) {
                    System.out.println("    " + dim + " -> " + values[dim]);
                }
            }
            docs[docID] = values;
            w.add(scratch, docID);
        }
        long indexFP;
        try (IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT)) {
            indexFP = w.finish(out);
        }
        try (IndexInput in = dir.openInput("bkd", IOContext.DEFAULT)) {
            in.seek(indexFP);
            BKDReader r = new BKDReader(in);
            int iters = atLeast(100);
            for (int iter = 0; iter < iters; iter++) {
                if (VERBOSE) {
                    System.out.println("\nTEST: iter=" + iter);
                }
                // Random N dims rect query:
                BigInteger[] queryMin = new BigInteger[numDims];
                BigInteger[] queryMax = new BigInteger[numDims];
                for (int dim = 0; dim < numDims; dim++) {
                    queryMin[dim] = randomBigInt(numBytesPerDim);
                    queryMax[dim] = randomBigInt(numBytesPerDim);
                    if (queryMin[dim].compareTo(queryMax[dim]) > 0) {
                        BigInteger x = queryMin[dim];
                        queryMin[dim] = queryMax[dim];
                        queryMax[dim] = x;
                    }
                }
                final BitSet hits = new BitSet();
                r.intersect(new IntersectVisitor() {

                    @Override
                    public void visit(int docID) {
                        hits.set(docID);
                    //System.out.println("visit docID=" + docID);
                    }

                    @Override
                    public void visit(int docID, byte[] packedValue) {
                        //System.out.println("visit check docID=" + docID);
                        for (int dim = 0; dim < numDims; dim++) {
                            BigInteger x = NumericUtils.sortableBytesToBigInt(packedValue, dim * numBytesPerDim, numBytesPerDim);
                            if (x.compareTo(queryMin[dim]) < 0 || x.compareTo(queryMax[dim]) > 0) {
                                //System.out.println("  no");
                                return;
                            }
                        }
                        //System.out.println("  yes");
                        hits.set(docID);
                    }

                    @Override
                    public Relation compare(byte[] minPacked, byte[] maxPacked) {
                        boolean crosses = false;
                        for (int dim = 0; dim < numDims; dim++) {
                            BigInteger min = NumericUtils.sortableBytesToBigInt(minPacked, dim * numBytesPerDim, numBytesPerDim);
                            BigInteger max = NumericUtils.sortableBytesToBigInt(maxPacked, dim * numBytesPerDim, numBytesPerDim);
                            assert max.compareTo(min) >= 0;
                            if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) {
                                return Relation.CELL_OUTSIDE_QUERY;
                            } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) {
                                crosses = true;
                            }
                        }
                        if (crosses) {
                            return Relation.CELL_CROSSES_QUERY;
                        } else {
                            return Relation.CELL_INSIDE_QUERY;
                        }
                    }
                });
                for (int docID = 0; docID < numDocs; docID++) {
                    BigInteger[] docValues = docs[docID];
                    boolean expected = true;
                    for (int dim = 0; dim < numDims; dim++) {
                        BigInteger x = docValues[dim];
                        if (x.compareTo(queryMin[dim]) < 0 || x.compareTo(queryMax[dim]) > 0) {
                            expected = false;
                            break;
                        }
                    }
                    boolean actual = hits.get(docID);
                    assertEquals("docID=" + docID, expected, actual);
                }
            }
        }
    }
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) BitSet(java.util.BitSet) CorruptingIndexOutput(org.apache.lucene.store.CorruptingIndexOutput) IndexOutput(org.apache.lucene.store.IndexOutput) Relation(org.apache.lucene.index.PointValues.Relation) BigInteger(java.math.BigInteger) IndexInput(org.apache.lucene.store.IndexInput) FilterDirectory(org.apache.lucene.store.FilterDirectory) Directory(org.apache.lucene.store.Directory)

Aggregations

BitSet (java.util.BitSet)2037 Test (org.junit.Test)294 ArrayList (java.util.ArrayList)224 List (java.util.List)83 HashMap (java.util.HashMap)76 Map (java.util.Map)70 IOException (java.io.IOException)60 HashSet (java.util.HashSet)52 Test (org.junit.jupiter.api.Test)47 RegisterSpecList (com.android.dx.rop.code.RegisterSpecList)45 Random (java.util.Random)42 Configuration (org.apache.hadoop.conf.Configuration)40 Path (org.apache.hadoop.fs.Path)39 ValidReadTxnList (org.apache.hadoop.hive.common.ValidReadTxnList)33 BlockNode (jadx.core.dex.nodes.BlockNode)29 File (java.io.File)27 LinkedList (java.util.LinkedList)27 RexNode (org.apache.calcite.rex.RexNode)27 ByteBuffer (java.nio.ByteBuffer)25 ValidReaderWriteIdList (org.apache.hadoop.hive.common.ValidReaderWriteIdList)25