Examples with FixedBitSet org.apache.lucene.util.FixedBitSet used on opensource projects

Example 1 with FixedBitSet

use of org.apache.lucene.util.FixedBitSet in project elasticsearch by elastic.

the class LocalCheckpointTracker method markSeqNoAsCompleted.

/**
     * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible.
     *
     * @param seqNo the sequence number to mark as completed
     */
public synchronized void markSeqNoAsCompleted(final long seqNo) {
    // make sure we track highest seen sequence number
    if (seqNo >= nextSeqNo) {
        nextSeqNo = seqNo + 1;
    }
    if (seqNo <= checkpoint) {
        // this is possible during recovery where we might replay an operation that was also replicated
        return;
    }
    final FixedBitSet bitSet = getBitSetForSeqNo(seqNo);
    final int offset = seqNoToBitSetOffset(seqNo);
    bitSet.set(offset);
    if (seqNo == checkpoint + 1) {
        updateCheckpoint();
    }
}

Example 2 with FixedBitSet

use of org.apache.lucene.util.FixedBitSet in project elasticsearch by elastic.

the class LocalCheckpointTracker method updateCheckpoint.

/**
     * Moves the checkpoint to the last consecutively processed sequence number. This method assumes that the sequence number following the
     * current checkpoint is processed.
     */
@SuppressForbidden(reason = "Object#notifyAll")
private void updateCheckpoint() {
    assert Thread.holdsLock(this);
    assert checkpoint < firstProcessedSeqNo + bitArraysSize - 1 : "checkpoint should be below the end of the first bit set (o.w. current bit set is completed and shouldn't be there)";
    assert getBitSetForSeqNo(checkpoint + 1) == processedSeqNo.getFirst() : "checkpoint + 1 doesn't point to the first bit set (o.w. current bit set is completed and shouldn't be there)";
    assert getBitSetForSeqNo(checkpoint + 1).get(seqNoToBitSetOffset(checkpoint + 1)) : "updateCheckpoint is called but the bit following the checkpoint is not set";
    try {
        // keep it simple for now, get the checkpoint one by one; in the future we can optimize and read words
        FixedBitSet current = processedSeqNo.getFirst();
        do {
            checkpoint++;
            // on the last bit of the current bit set, we can clean it.
            if (checkpoint == firstProcessedSeqNo + bitArraysSize - 1) {
                processedSeqNo.removeFirst();
                firstProcessedSeqNo += bitArraysSize;
                assert checkpoint - firstProcessedSeqNo < bitArraysSize;
                current = processedSeqNo.peekFirst();
            }
        } while (current != null && current.get(seqNoToBitSetOffset(checkpoint + 1)));
    } finally {
        // notifies waiters in waitForOpsToComplete
        this.notifyAll();
    }
}

Example 3 with FixedBitSet

use of org.apache.lucene.util.FixedBitSet in project elasticsearch by elastic.

the class AbstractStringFieldDataTestCase method testNestedSorting.

public void testNestedSorting(MultiValueMode sortMode) throws IOException {
    final String[] values = new String[randomIntBetween(2, 20)];
    for (int i = 0; i < values.length; ++i) {
        values[i] = TestUtil.randomSimpleString(random());
    }
    final int numParents = scaledRandomIntBetween(10, 3072);
    List<Document> docs = new ArrayList<>();
    FixedBitSet parents = new FixedBitSet(64);
    for (int i = 0; i < numParents; ++i) {
        docs.clear();
        final int numChildren = randomInt(4);
        for (int j = 0; j < numChildren; ++j) {
            final Document child = new Document();
            final int numValues = randomInt(3);
            for (int k = 0; k < numValues; ++k) {
                final String value = RandomPicks.randomFrom(random(), values);
                addField(child, "text", value);
            }
            docs.add(child);
        }
        final Document parent = new Document();
        parent.add(new StringField("type", "parent", Store.YES));
        final String value = RandomPicks.randomFrom(random(), values);
        if (value != null) {
            addField(parent, "text", value);
        }
        docs.add(parent);
        int bit = parents.prevSetBit(parents.length() - 1) + docs.size();
        parents = FixedBitSet.ensureCapacity(parents, bit);
        parents.set(bit);
        writer.addDocuments(docs);
        if (randomInt(10) == 0) {
            writer.commit();
        }
    }
    DirectoryReader directoryReader = DirectoryReader.open(writer);
    directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0));
    IndexSearcher searcher = new IndexSearcher(directoryReader);
    IndexFieldData<?> fieldData = getForField("text");
    final Object missingValue;
    switch(randomInt(4)) {
        case 0:
            missingValue = "_first";
            break;
        case 1:
            missingValue = "_last";
            break;
        case 2:
            missingValue = new BytesRef(RandomPicks.randomFrom(random(), values));
            break;
        default:
            missingValue = new BytesRef(TestUtil.randomSimpleString(random()));
            break;
    }
    Query parentFilter = new TermQuery(new Term("type", "parent"));
    Query childFilter = Queries.not(parentFilter);
    Nested nested = createNested(searcher, parentFilter, childFilter);
    BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested);
    ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None);
    Sort sort = new Sort(new SortField("text", nestedComparatorSource));
    TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
    assertTrue(topDocs.scoreDocs.length > 0);
    BytesRef previous = null;
    for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
        final int docID = topDocs.scoreDocs[i].doc;
        assertTrue("expected " + docID + " to be a parent", parents.get(docID));
        BytesRef cmpValue = null;
        for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
            String[] sVals = searcher.doc(child).getValues("text");
            final BytesRef[] vals;
            if (sVals.length == 0) {
                vals = new BytesRef[0];
            } else {
                vals = new BytesRef[sVals.length];
                for (int j = 0; j < vals.length; ++j) {
                    vals[j] = new BytesRef(sVals[j]);
                }
            }
            for (BytesRef value : vals) {
                if (cmpValue == null) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) {
                    cmpValue = value;
                }
            }
        }
        if (cmpValue == null) {
            if ("_first".equals(missingValue)) {
                cmpValue = new BytesRef();
            } else if ("_last".equals(missingValue) == false) {
                cmpValue = (BytesRef) missingValue;
            }
        }
        if (previous != null && cmpValue != null) {
            assertTrue(previous.utf8ToString() + "   /   " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
        }
        previous = cmpValue;
    }
    searcher.getIndexReader().close();
}

Example 4 with FixedBitSet

use of org.apache.lucene.util.FixedBitSet in project elasticsearch by elastic.

the class InternalEngineTests method getSeqNosSet.

private static FixedBitSet getSeqNosSet(final IndexReader reader, final long highestSeqNo) throws IOException {
    // _seq_no are stored as doc values for the time being, so this is how we get them
    // (as opposed to using an IndexSearcher or IndexReader)
    final FixedBitSet bitSet = new FixedBitSet((int) highestSeqNo + 1);
    final List<LeafReaderContext> leaves = reader.leaves();
    if (leaves.isEmpty()) {
        return bitSet;
    }
    for (int i = 0; i < leaves.size(); i++) {
        final LeafReader leaf = leaves.get(i).reader();
        final NumericDocValues values = leaf.getNumericDocValues(SeqNoFieldMapper.NAME);
        if (values == null) {
            continue;
        }
        final Bits bits = leaf.getLiveDocs();
        for (int docID = 0; docID < leaf.maxDoc(); docID++) {
            if (bits == null || bits.get(docID)) {
                final long seqNo = values.get(docID);
                assertFalse("should not have more than one document with the same seq_no[" + seqNo + "]", bitSet.get((int) seqNo));
                bitSet.set((int) seqNo);
            }
        }
    }
    return bitSet;
}

Example 5 with FixedBitSet

use of org.apache.lucene.util.FixedBitSet in project elasticsearch by elastic.

the class MultiValueModeTests method testUnsortedSingleValuedDoubles.

public void testUnsortedSingleValuedDoubles() throws Exception {
    final int numDocs = scaledRandomIntBetween(1, 100);
    final double[] array = new double[numDocs];
    final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
    for (int i = 0; i < array.length; ++i) {
        if (randomBoolean()) {
            array[i] = randomDouble();
            if (docsWithValue != null) {
                docsWithValue.set(i);
            }
        } else if (docsWithValue != null && randomBoolean()) {
            docsWithValue.set(i);
        }
    }
    final NumericDoubleValues singleValues = new NumericDoubleValues() {

        @Override
        public double get(int docID) {
            return array[docID];
        }
    };
    final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues, docsWithValue);
    final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() {

        @Override
        public int count() {
            return singletonValues.count();
        }

        @Override
        public void setDocument(int doc) {
            singletonValues.setDocument(doc);
        }

        @Override
        public double valueAt(int index) {
            return Math.cos(singletonValues.valueAt(index));
        }
    };
    verify(multiValues, numDocs);
}