Examples with DocIdSetBlock com.linkedin.pinot.core.operator.blocks.DocIdSetBlock used on opensource projects

Example 1 with DocIdSetBlock

use of com.linkedin.pinot.core.operator.blocks.DocIdSetBlock in project pinot by linkedin.

the class MSelectionOnlyOperator method getNextBlock.

@Override
public Block getNextBlock() {
    int numDocsScanned = 0;
    ProjectionBlock projectionBlock;
    while ((projectionBlock = (ProjectionBlock) _projectionOperator.nextBlock()) != null) {
        for (int i = 0; i < _dataSchema.size(); i++) {
            _blocks[i] = projectionBlock.getBlock(_dataSchema.getColumnName(i));
        }
        SelectionFetcher selectionFetcher = new SelectionFetcher(_blocks, _dataSchema);
        DocIdSetBlock docIdSetBlock = projectionBlock.getDocIdSetBlock();
        int numDocsToFetch = Math.min(docIdSetBlock.getSearchableLength(), _limitDocs - _rowEvents.size());
        numDocsScanned += numDocsToFetch;
        int[] docIdSet = docIdSetBlock.getDocIdSet();
        for (int i = 0; i < numDocsToFetch; i++) {
            _rowEvents.add(selectionFetcher.getRow(docIdSet[i]));
        }
        if (_rowEvents.size() == _limitDocs) {
            break;
        }
    }
    // Create execution statistics.
    long numEntriesScannedInFilter = _projectionOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
    long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumProjectionColumns();
    long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs);
    return new IntermediateResultsBlock(_dataSchema, _rowEvents);
}

Example 2 with DocIdSetBlock

use of com.linkedin.pinot.core.operator.blocks.DocIdSetBlock in project pinot by linkedin.

the class DefaultGroupKeyGeneratorTest method setup.

@BeforeClass
private void setup() throws Exception {
    GenericRow[] segmentData = new GenericRow[NUM_ROWS];
    int value = _random.nextInt(MAX_STEP_LENGTH);
    // Generate random values for the segment.
    for (int i = 0; i < UNIQUE_ROWS; i++) {
        Map<String, Object> map = new HashMap<>();
        for (String singleValueColumn : SINGLE_VALUE_COLUMNS) {
            map.put(singleValueColumn, value);
            value += 1 + _random.nextInt(MAX_STEP_LENGTH);
        }
        for (String multiValueColumn : MULTI_VALUE_COLUMNS) {
            int numMultiValues = 1 + _random.nextInt(MAX_NUM_MULTI_VALUES);
            Integer[] values = new Integer[numMultiValues];
            for (int k = 0; k < numMultiValues; k++) {
                values[k] = value;
                value += 1 + _random.nextInt(MAX_STEP_LENGTH);
            }
            map.put(multiValueColumn, values);
        }
        GenericRow genericRow = new GenericRow();
        genericRow.init(map);
        segmentData[i] = genericRow;
    }
    for (int i = UNIQUE_ROWS; i < NUM_ROWS; i += UNIQUE_ROWS) {
        System.arraycopy(segmentData, 0, segmentData, i, UNIQUE_ROWS);
    }
    // Create an index segment with the random values.
    Schema schema = new Schema();
    for (String singleValueColumn : SINGLE_VALUE_COLUMNS) {
        DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(singleValueColumn, FieldSpec.DataType.INT, true);
        schema.addField(dimensionFieldSpec);
    }
    for (String multiValueColumn : MULTI_VALUE_COLUMNS) {
        DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(multiValueColumn, FieldSpec.DataType.INT, false);
        schema.addField(dimensionFieldSpec);
    }
    SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
    FileUtils.deleteQuietly(new File(INDEX_DIR_PATH));
    config.setOutDir(INDEX_DIR_PATH);
    config.setSegmentName(SEGMENT_NAME);
    SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
    driver.init(config, new TestDataRecordReader(schema, segmentData));
    driver.build();
    IndexSegment indexSegment = Loaders.IndexSegment.load(new File(INDEX_DIR_PATH, SEGMENT_NAME), ReadMode.heap);
    // Get a data fetcher for the index segment.
    Map<String, BaseOperator> dataSourceMap = new HashMap<>();
    Map<String, Block> blockMap = new HashMap<>();
    for (String column : indexSegment.getColumnNames()) {
        DataSource dataSource = indexSegment.getDataSource(column);
        dataSourceMap.put(column, dataSource);
        blockMap.put(column, dataSource.getNextBlock());
    }
    // Generate a random test doc id set.
    int num1 = _random.nextInt(50);
    int num2 = num1 + 1 + _random.nextInt(50);
    for (int i = 0; i < 20; i += 2) {
        _testDocIdSet[i] = num1 + 50 * i;
        _testDocIdSet[i + 1] = num2 + 50 * i;
    }
    DataFetcher dataFetcher = new DataFetcher(dataSourceMap);
    DocIdSetBlock docIdSetBlock = new DocIdSetBlock(_testDocIdSet, _testDocIdSet.length);
    ProjectionBlock projectionBlock = new ProjectionBlock(blockMap, new DataBlockCache(dataFetcher), docIdSetBlock);
    _transformBlock = new TransformBlock(projectionBlock, new HashMap<String, BlockValSet>());
}

Example 3 with DocIdSetBlock

use of com.linkedin.pinot.core.operator.blocks.DocIdSetBlock in project pinot by linkedin.

the class MSelectionOrderByOperator method getNextBlock.

@Override
public Block getNextBlock() {
    int numDocsScanned = 0;
    ProjectionBlock projectionBlock;
    while ((projectionBlock = (ProjectionBlock) _projectionOperator.nextBlock()) != null) {
        for (int i = 0; i < _dataSchema.size(); i++) {
            _blocks[i] = projectionBlock.getBlock(_dataSchema.getColumnName(i));
        }
        DocIdSetBlock docIdSetBlock = projectionBlock.getDocIdSetBlock();
        _selectionOperatorService.iterateOnBlocksWithOrdering(docIdSetBlock.getBlockDocIdSet().iterator(), _blocks);
    }
    // Create execution statistics.
    numDocsScanned += _selectionOperatorService.getNumDocsScanned();
    long numEntriesScannedInFilter = _projectionOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
    long numEntriesScannedPostFilter = numDocsScanned * _projectionOperator.getNumProjectionColumns();
    long numTotalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, numTotalRawDocs);
    return new IntermediateResultsBlock(_selectionOperatorService.getDataSchema(), _selectionOperatorService.getRows());
}

Example 4 with DocIdSetBlock

use of com.linkedin.pinot.core.operator.blocks.DocIdSetBlock in project pinot by linkedin.

the class MProjectionOperator method getNextBlock.

@Override
public ProjectionBlock getNextBlock() {
    DocIdSetBlock docIdSetBlock = (DocIdSetBlock) _docIdSetOperator.nextBlock();
    if (docIdSetBlock == null) {
        _currentBlock = null;
    } else {
        _blockMap.put("_docIdSet", docIdSetBlock);
        for (String column : _columnToDataSourceMap.keySet()) {
            _blockMap.put(column, _columnToDataSourceMap.get(column).nextBlock(new BlockId(0)));
        }
        _currentBlock = new ProjectionBlock(_blockMap, _dataBlockCache, docIdSetBlock);
    }
    return _currentBlock;
}