Examples with BlockMultiValIterator com.linkedin.pinot.core.common.BlockMultiValIterator used on opensource projects

Example 1 with BlockMultiValIterator

use of com.linkedin.pinot.core.common.BlockMultiValIterator in project pinot by linkedin.

the class RealtimeTableDataManagerTest method testSetup.

public void testSetup() throws Exception {
    final HLRealtimeSegmentDataManager manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, null, tableDataManagerConfig.getDataDir(), ReadMode.valueOf(tableDataManagerConfig.getReadMode()), getTestSchema(), new ServerMetrics(new MetricsRegistry()));
    final long start = System.currentTimeMillis();
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            if (System.currentTimeMillis() - start >= (SEGMENT_CONSUMING_TIME)) {
                keepOnRunning = false;
            }
        }
    }, 1000, 1000 * 60 * 1);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("count");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("count column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan metric col count : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerId");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("viewerId column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV dimension col viewerId : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("daysSinceEpoch");
                BlockValSet valSet = mDs.nextBlock().getBlockValueSet();
                BlockSingleValIterator valIt = (BlockSingleValIterator) valSet.iterator();
                int val = valIt.nextIntVal();
                while (val != Constants.EOF) {
                    val = valIt.nextIntVal();
                    sum += val;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan SV time col daysSinceEpoch : " + (stop - start) + " sum : " + sum);
        }
    }, 20000, 1000 * 5);
    TimerService.timer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            long start = System.currentTimeMillis();
            long sum = 0;
            float sumOfLengths = 0F;
            float counter = 0F;
            try {
                RealtimeSegment segment = (RealtimeSegment) manager.getSegment();
                RealtimeColumnDataSource mDs = (RealtimeColumnDataSource) segment.getDataSource("viewerCompanies");
                Block b = mDs.nextBlock();
                BlockValSet valSet = b.getBlockValueSet();
                BlockMultiValIterator valIt = (BlockMultiValIterator) valSet.iterator();
                BlockMetadata m = b.getMetadata();
                int maxVams = m.getMaxNumberOfMultiValues();
                while (valIt.hasNext()) {
                    int[] vals = new int[maxVams];
                    int len = valIt.nextIntVal(vals);
                    for (int i = 0; i < len; i++) {
                        sum += vals[i];
                    }
                    sumOfLengths += len;
                    counter++;
                }
            } catch (Exception e) {
                LOGGER.info("daysSinceEpoch column exception");
                e.printStackTrace();
            }
            long stop = System.currentTimeMillis();
            LOGGER.info("time to scan MV col viewerCompanies : " + (stop - start) + " sum : " + sum + " average len : " + (sumOfLengths / counter));
        }
    }, 20000, 1000 * 5);
    while (keepOnRunning) {
    // Wait for keepOnRunning to be set to false
    }
}

Example 2 with BlockMultiValIterator

use of com.linkedin.pinot.core.common.BlockMultiValIterator in project pinot by linkedin.

the class SegmentQueryProcessor method evaluatePredicate.

private List<Integer> evaluatePredicate(List<Integer> inputDocIds, String column, PredicateFilter predicateFilter) {
    List<Integer> result = new ArrayList<>();
    if (!_mvColumns.contains(column)) {
        BlockSingleValIterator bvIter = (BlockSingleValIterator) _indexSegment.getDataSource(column).getNextBlock().getBlockValueSet().iterator();
        int i = 0;
        while (bvIter.hasNext() && (inputDocIds == null || i < inputDocIds.size())) {
            int docId = (inputDocIds != null) ? inputDocIds.get(i++) : i++;
            bvIter.skipTo(docId);
            if (predicateFilter.apply(bvIter.nextIntVal())) {
                result.add(docId);
            }
        }
    } else {
        BlockMultiValIterator bvIter = (BlockMultiValIterator) _indexSegment.getDataSource(column).getNextBlock().getBlockValueSet().iterator();
        int i = 0;
        while (bvIter.hasNext() && (inputDocIds == null || i < inputDocIds.size())) {
            int docId = (inputDocIds != null) ? inputDocIds.get(i++) : i++;
            bvIter.skipTo(docId);
            int[] dictIds = _mvColumnArrayMap.get(column);
            int numMVValues = bvIter.nextIntVal(dictIds);
            if (predicateFilter.apply(dictIds, numMVValues)) {
                result.add(docId);
            }
        }
    }
    return result;
}

Example 3 with BlockMultiValIterator

use of com.linkedin.pinot.core.common.BlockMultiValIterator in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test3.

@Test
public void test3() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final DataSource ds = segment.getDataSource("column7");
    final Block bl = ds.nextBlock();
    final BlockValSet valSet = bl.getBlockValueSet();
    final int maxValue = ((SegmentMetadataImpl) segment.getSegmentMetadata()).getColumnMetadataFor("column7").getMaxNumberOfMultiValues();
    final BlockMultiValIterator it = (BlockMultiValIterator) valSet.iterator();
    while (it.hasNext()) {
        final int[] entry = new int[maxValue];
        it.nextIntVal(entry);
        LOGGER.trace(Arrays.toString(entry));
    }
}

Example 4 with BlockMultiValIterator

use of com.linkedin.pinot.core.common.BlockMultiValIterator in project pinot by linkedin.

the class Projection method run.

public ResultTable run() {
    ResultTable resultTable = new ResultTable(_columnList, _filteredDocIds.size());
    resultTable.setResultType(ResultTable.ResultType.Selection);
    for (Pair pair : _columnList) {
        String column = (String) pair.getFirst();
        if (!_mvColumns.contains(column)) {
            BlockSingleValIterator bvIter = (BlockSingleValIterator) _indexSegment.getDataSource(column).getNextBlock().getBlockValueSet().iterator();
            int rowId = 0;
            for (Integer docId : _filteredDocIds) {
                bvIter.skipTo(docId);
                resultTable.add(rowId++, bvIter.nextIntVal());
            }
        } else {
            BlockMultiValIterator bvIter = (BlockMultiValIterator) _indexSegment.getDataSource(column).getNextBlock().getBlockValueSet().iterator();
            int rowId = 0;
            for (Integer docId : _filteredDocIds) {
                bvIter.skipTo(docId);
                int[] dictIds = _mvColumnArrayMap.get(column);
                int numMVValues = bvIter.nextIntVal(dictIds);
                dictIds = Arrays.copyOf(dictIds, numMVValues);
                resultTable.add(rowId++, ArrayUtils.toObject(dictIds));
            }
        }
    }
    return transformFromIdToValues(resultTable, _dictionaryMap, _addCountStar);
}

Example 5 with BlockMultiValIterator

use of com.linkedin.pinot.core.common.BlockMultiValIterator in project pinot by linkedin.

the class IndexSegmentImpl method iterator.

public Iterator<GenericRow> iterator(final int startDocId, final int endDocId) {
    final Map<String, BlockSingleValIterator> singleValIteratorMap = new HashMap<>();
    final Map<String, BlockMultiValIterator> multiValIteratorMap = new HashMap<>();
    for (String column : getColumnNames()) {
        DataSource dataSource = getDataSource(column);
        BlockValIterator iterator = dataSource.getNextBlock().getBlockValueSet().iterator();
        if (dataSource.getDataSourceMetadata().isSingleValue()) {
            singleValIteratorMap.put(column, (BlockSingleValIterator) iterator);
        } else {
            multiValIteratorMap.put(column, (BlockMultiValIterator) iterator);
        }
    }
    return new Iterator<GenericRow>() {

        int docId = startDocId;

        @Override
        public boolean hasNext() {
            return docId < endDocId;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

        @Override
        public GenericRow next() {
            Map<String, Object> map = new HashMap<>();
            for (String column : singleValIteratorMap.keySet()) {
                int dictId = singleValIteratorMap.get(column).nextIntVal();
                Dictionary dictionary = getDictionaryFor(column);
                map.put(column, dictionary.get(dictId));
            }
            for (String column : multiValIteratorMap.keySet()) {
            //TODO:handle multi value
            }
            GenericRow genericRow = new GenericRow();
            genericRow.init(map);
            docId++;
            return genericRow;
        }
    };
}