Examples with BlockDocIdIterator com.linkedin.pinot.core.common.BlockDocIdIterator used on opensource projects

Example 21 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class RealtimeSegmentTest method testRangeMatchFilteringMetricPredicateWithInvIdx.

@Test
public void testRangeMatchFilteringMetricPredicateWithInvIdx() throws Exception {
    DataSource ds1 = segmentWithInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t*)");
    Predicate predicate = new RangePredicate("count", rhs);
    BitmapBasedFilterOperator op = new BitmapBasedFilterOperator(predicate, ds1, 0, segmentWithInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    DataSource ds2 = segmentWithInvIdx.getDataSource("count");
    BlockSingleValIterator blockValIterator = (BlockSingleValIterator) ds2.nextBlock().getBlockValueSet().iterator();
    int docId = iterator.next();
    int counter = 0;
    while (docId != Constants.EOF) {
        blockValIterator.skipTo(docId);
        Assert.assertEquals(ds1.getDictionary().get(blockValIterator.nextIntVal()), 890662862);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 100000);
}

Example 22 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class BitmapDocIdIteratorTest method testIt.

@Test
public void testIt() {
    MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
    bitmap.add(0);
    bitmap.add(1);
    bitmap.add(3);
    bitmap.add(4);
    // Iterate over all values
    int[] expected = new int[] { 0, 1, 3, 4, Constants.EOF, Constants.EOF };
    checkDocIdIterator(expected, new BitmapDocIdIterator(bitmap.getIntIterator()));
    checkDocIdIterator(expected, new RangelessBitmapDocIdIterator(bitmap.getIntIterator()));
    // Check that advancing to an existing value works
    expected = new int[] { 4, Constants.EOF, Constants.EOF };
    BlockDocIdIterator docIdIterator = new BitmapDocIdIterator(bitmap.getIntIterator());
    Assert.assertEquals(docIdIterator.advance(3), 3);
    checkDocIdIterator(expected, docIdIterator);
    docIdIterator = new BitmapDocIdIterator(bitmap.getIntIterator());
    Assert.assertEquals(docIdIterator.advance(3), 3);
    checkDocIdIterator(expected, docIdIterator);
    // Check that advancing to a value that does not exist works
    expected = new int[] { 4, Constants.EOF, Constants.EOF };
    docIdIterator = new BitmapDocIdIterator(bitmap.getIntIterator());
    Assert.assertEquals(docIdIterator.advance(2), 3);
    checkDocIdIterator(expected, docIdIterator);
    docIdIterator = new BitmapDocIdIterator(bitmap.getIntIterator());
    Assert.assertEquals(docIdIterator.advance(2), 3);
    checkDocIdIterator(expected, docIdIterator);
}

Example 23 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class AndOperatorTest method testIntersectionForTwoLists.

@Test
public void testIntersectionForTwoLists() {
    int[] list1 = new int[] { 2, 3, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final AndOperator andOperator = new AndOperator(operators);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}

Example 24 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class OrOperatorTest method testIntersectionForTwoLists.

@Test
public void testIntersectionForTwoLists() {
    int[] list1 = new int[] { 2, 3, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final OrOperator orOperator = new OrOperator(operators);
    orOperator.open();
    Block block;
    TreeSet<Integer> set = new TreeSet<Integer>();
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list1)));
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list2)));
    Iterator<Integer> expectedIterator = set.iterator();
    while ((block = orOperator.nextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
            Assert.assertEquals(expectedIterator.next().intValue(), docId);
        }
    }
    orOperator.close();
}

Example 25 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class OrOperatorTest method testIntersectionForThreeLists.

@Test
public void testIntersectionForThreeLists() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    operators.add(makeFilterOperator(list3));
    final OrOperator orOperator = new OrOperator(operators);
    orOperator.open();
    Block block;
    TreeSet<Integer> set = new TreeSet<Integer>();
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list1)));
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list2)));
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list3)));
    Iterator<Integer> expectedIterator = set.iterator();
    while ((block = orOperator.nextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
            Assert.assertEquals(expectedIterator.next().intValue(), docId);
        }
    }
    orOperator.close();
}