Search in sources :

Example 6 with BlockSingleValIterator

use of com.linkedin.pinot.core.common.BlockSingleValIterator in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method testDataSourceWithoutPredicateForSingleValueDimensionColumns.

private void testDataSourceWithoutPredicateForSingleValueDimensionColumns() {
    for (FieldSpec spec : schema.getAllFieldSpecs()) {
        if (spec.isSingleValueField() && spec.getFieldType() == FieldType.DIMENSION) {
            DataSource offlineDS = offlineSegment.getDataSource(spec.getName());
            DataSource realtimeDS = realtimeSegment.getDataSource(spec.getName());
            Block offlineBlock = offlineDS.nextBlock();
            Block realtimeBlock = realtimeDS.nextBlock();
            BlockMetadata offlineMetadata = offlineBlock.getMetadata();
            BlockMetadata realtimeMetadata = realtimeBlock.getMetadata();
            BlockSingleValIterator offlineValIterator = (BlockSingleValIterator) offlineBlock.getBlockValueSet().iterator();
            BlockSingleValIterator realtimeValIterator = (BlockSingleValIterator) realtimeBlock.getBlockValueSet().iterator();
            Assert.assertEquals(offlineSegment.getSegmentMetadata().getTotalDocs(), realtimeSegment.getAggregateDocumentCount());
            while (realtimeValIterator.hasNext()) {
                int offlineDicId = offlineValIterator.nextIntVal();
                int realtimeDicId = realtimeValIterator.nextIntVal();
                try {
                    Assert.assertEquals(offlineMetadata.getDictionary().get(offlineDicId), realtimeMetadata.getDictionary().get(realtimeDicId));
                } catch (AssertionError e) {
                    LOGGER.info("column : {}", spec.getName());
                    LOGGER.info("realtimeDicId : {}, rawValue : {}", realtimeDicId, realtimeMetadata.getDictionary().get(realtimeDicId));
                    LOGGER.info("offlineDicId : {}, rawValue : {}", offlineDicId, offlineMetadata.getDictionary().get(offlineDicId));
                    throw e;
                }
            }
            Assert.assertEquals(offlineValIterator.hasNext(), realtimeValIterator.hasNext());
        }
    }
}
Also used : BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) BlockMetadata(com.linkedin.pinot.core.common.BlockMetadata) Block(com.linkedin.pinot.core.common.Block) FieldSpec(com.linkedin.pinot.common.data.FieldSpec) DataSource(com.linkedin.pinot.core.common.DataSource)

Example 7 with BlockSingleValIterator

use of com.linkedin.pinot.core.common.BlockSingleValIterator in project pinot by linkedin.

the class RealtimeSegmentTest method testMetricPredicateWithInvIdx.

@Test
public void testMetricPredicateWithInvIdx() throws Exception {
    DataSource ds1 = segmentWithInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("890662862");
    Predicate predicate = new EqPredicate("count", rhs);
    BitmapBasedFilterOperator op = new BitmapBasedFilterOperator(predicate, ds1, 0, segmentWithInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    DataSource ds2 = segmentWithInvIdx.getDataSource("count");
    BlockSingleValIterator blockValIterator = (BlockSingleValIterator) ds2.nextBlock().getBlockValueSet().iterator();
    int docId = iterator.next();
    int counter = 0;
    while (docId != Constants.EOF) {
        blockValIterator.skipTo(docId);
        Assert.assertEquals(ds1.getDictionary().get(blockValIterator.nextIntVal()), 890662862);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 100000);
}
Also used : BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) DataSource(com.linkedin.pinot.core.common.DataSource) RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Example 8 with BlockSingleValIterator

use of com.linkedin.pinot.core.common.BlockSingleValIterator in project pinot by linkedin.

the class RealtimeSegmentTest method testRangeMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testRangeMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t*)");
    Predicate predicate = new RangePredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    DataSource ds2 = segmentWithoutInvIdx.getDataSource("count");
    BlockSingleValIterator blockValIterator = (BlockSingleValIterator) ds2.nextBlock().getBlockValueSet().iterator();
    int docId = iterator.next();
    int counter = 0;
    while (docId != Constants.EOF) {
        blockValIterator.skipTo(docId);
        Assert.assertEquals(ds1.getDictionary().get(blockValIterator.nextIntVal()), 890662862);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 100000);
}
Also used : RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) DataSource(com.linkedin.pinot.core.common.DataSource) RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Example 9 with BlockSingleValIterator

use of com.linkedin.pinot.core.common.BlockSingleValIterator in project pinot by linkedin.

the class RealtimeSegmentTest method test2.

@Test
public void test2() throws Exception {
    DataSource ds = segmentWithoutInvIdx.getDataSource("column1");
    Block b = ds.nextBlock();
    BlockValSet set = b.getBlockValueSet();
    BlockSingleValIterator it = (BlockSingleValIterator) set.iterator();
    BlockMetadata metadata = b.getMetadata();
    while (it.next()) {
        int dicId = it.nextIntVal();
    }
}
Also used : BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) BlockMetadata(com.linkedin.pinot.core.common.BlockMetadata) Block(com.linkedin.pinot.core.common.Block) BlockValSet(com.linkedin.pinot.core.common.BlockValSet) DataSource(com.linkedin.pinot.core.common.DataSource) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Example 10 with BlockSingleValIterator

use of com.linkedin.pinot.core.common.BlockSingleValIterator in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test2.

@Test
public void test2() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    //    System.out.println("INdex dir:" + INDEX_DIR);
    final DataSource ds = segment.getDataSource("column1");
    final Block bl = ds.nextBlock();
    final BlockValSet valSet = bl.getBlockValueSet();
    final BlockSingleValIterator it = (BlockSingleValIterator) valSet.iterator();
    // TODO: FIXME - load segment with known data and verify that it exists
    while (it.hasNext()) {
        LOGGER.trace(Integer.toString(it.nextIntVal()));
    }
}
Also used : IndexSegmentImpl(com.linkedin.pinot.core.segment.index.IndexSegmentImpl) BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) Block(com.linkedin.pinot.core.common.Block) BlockValSet(com.linkedin.pinot.core.common.BlockValSet) DataSource(com.linkedin.pinot.core.common.DataSource) Test(org.testng.annotations.Test)

Aggregations

BlockSingleValIterator (com.linkedin.pinot.core.common.BlockSingleValIterator)17 DataSource (com.linkedin.pinot.core.common.DataSource)14 Block (com.linkedin.pinot.core.common.Block)12 Test (org.testng.annotations.Test)7 BlockMetadata (com.linkedin.pinot.core.common.BlockMetadata)6 RealtimeSegmentImplTest (com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)6 FieldSpec (com.linkedin.pinot.common.data.FieldSpec)5 BlockValSet (com.linkedin.pinot.core.common.BlockValSet)5 ArrayList (java.util.ArrayList)5 BlockDocIdIterator (com.linkedin.pinot.core.common.BlockDocIdIterator)4 BlockMultiValIterator (com.linkedin.pinot.core.common.BlockMultiValIterator)4 Predicate (com.linkedin.pinot.core.common.Predicate)4 EqPredicate (com.linkedin.pinot.core.common.predicate.EqPredicate)4 NEqPredicate (com.linkedin.pinot.core.common.predicate.NEqPredicate)4 RangePredicate (com.linkedin.pinot.core.common.predicate.RangePredicate)4 Dictionary (com.linkedin.pinot.core.segment.index.readers.Dictionary)4 HashMap (java.util.HashMap)3 BitmapBasedFilterOperator (com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator)2 ScanBasedFilterOperator (com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator)2 ServerMetrics (com.linkedin.pinot.common.metrics.ServerMetrics)1