Examples with BlockDocIdSet com.linkedin.pinot.core.common.BlockDocIdSet used on opensource projects

Example 6 with BlockDocIdSet

use of com.linkedin.pinot.core.common.BlockDocIdSet in project pinot by linkedin.

the class AndOperatorTest method testComplex.

@Test
public void testComplex() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final AndOperator andOperator1 = new AndOperator(operators);
    List<BaseFilterOperator> operators1 = new ArrayList<>();
    operators1.add(andOperator1);
    operators1.add(makeFilterOperator(list3));
    final AndOperator andOperator = new AndOperator(operators1);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}

Example 7 with BlockDocIdSet

use of com.linkedin.pinot.core.common.BlockDocIdSet in project pinot by linkedin.

the class OrOperatorTest method testComplex.

@Test
public void testComplex() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    TreeSet<Integer> set = new TreeSet<Integer>();
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list1)));
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list2)));
    set.addAll(Lists.newArrayList(ArrayUtils.toObject(list3)));
    Iterator<Integer> expectedIterator = set.iterator();
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final OrOperator orOperator1 = new OrOperator(operators);
    List<BaseFilterOperator> operators1 = new ArrayList<>();
    operators1.add(orOperator1);
    operators1.add(makeFilterOperator(list3));
    final OrOperator orOperator = new OrOperator(operators1);
    orOperator.open();
    BaseFilterBlock block;
    while ((block = orOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
            //        System.out.println(docId);
            Assert.assertEquals(expectedIterator.next().intValue(), docId);
        }
    }
    orOperator.close();
}

Example 8 with BlockDocIdSet

use of com.linkedin.pinot.core.common.BlockDocIdSet in project pinot by linkedin.

the class FilterTreeOptimizationTest method testQuery.

/**
   * Helper method to perform the actual testing for the given query.
   * <ul>
   *   <li> Constructs the operator tree with and without alwaysFalse optimizations. </li>
   *   <li> Compares that all docIds filtered by the two operators are identical. </li>
   * </ul>
   * @param query Query to run.
   */
private void testQuery(String query) {
    BrokerRequest brokerRequest = _compiler.compileToBrokerRequest(query);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    BaseFilterOperator expectedOperator = FilterPlanNode.constructPhysicalOperator(filterQueryTree, _indexSegment, false);
    BaseFilterOperator actualOperator = FilterPlanNode.constructPhysicalOperator(filterQueryTree, _indexSegment, true);
    BaseFilterBlock expectedBlock;
    while ((expectedBlock = expectedOperator.getNextBlock()) != null) {
        BaseFilterBlock actualBlock = actualOperator.getNextBlock();
        Assert.assertNotNull(actualBlock);
        final BlockDocIdSet expectedDocIdSet = expectedBlock.getBlockDocIdSet();
        final BlockDocIdIterator expectedIterator = expectedDocIdSet.iterator();
        final BlockDocIdSet actualDocIdSet = actualBlock.getBlockDocIdSet();
        final BlockDocIdIterator actualIterator = actualDocIdSet.iterator();
        int expectedDocId;
        int actualDocId;
        while (((expectedDocId = expectedIterator.next()) != Constants.EOF) && ((actualDocId = actualIterator.next()) != Constants.EOF)) {
            Assert.assertEquals(actualDocId, expectedDocId);
        }
        Assert.assertTrue(expectedIterator.next() == Constants.EOF);
        Assert.assertTrue(actualIterator.next() == Constants.EOF);
    }
}

Example 9 with BlockDocIdSet

use of com.linkedin.pinot.core.common.BlockDocIdSet in project pinot by linkedin.

the class OperatorSize method size.

public static int size(Operator operator) {
    Block block;
    int count = 0;
    operator.open();
    while ((block = operator.nextBlock()) != null) {
        BlockDocIdSet docIdSet = block.getBlockDocIdSet();
        count += size(docIdSet);
    }
    operator.close();
    System.out.println("count = " + count);
    return count;
}

Example 10 with BlockDocIdSet

use of com.linkedin.pinot.core.common.BlockDocIdSet in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test5.

@Test(enabled = false)
public void test5() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final List<String> rhs = new ArrayList<String>();
    rhs.add("-100");
    final Predicate p = new EqPredicate("column1", rhs);
    final DataSource ds = segment.getDataSource("column1", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}