Examples with Block com.linkedin.pinot.core.common.Block used on opensource projects

Example 26 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class BitmapBasedFilterOperator method nextFilterBlock.

@Override
public BaseFilterBlock nextFilterBlock(BlockId BlockId) {
    InvertedIndexReader invertedIndex = dataSource.getInvertedIndex();
    Block dataSourceBlock = dataSource.nextBlock();
    int[] dictionaryIds;
    boolean exclusion = false;
    switch(predicate.getType()) {
        case EQ:
        case IN:
        case RANGE:
            dictionaryIds = predicateEvaluator.getMatchingDictionaryIds();
            break;
        case NEQ:
        case NOT_IN:
            exclusion = true;
            dictionaryIds = predicateEvaluator.getNonMatchingDictionaryIds();
            break;
        case REGEX:
        default:
            throw new UnsupportedOperationException("Regex is not supported");
    }
    ImmutableRoaringBitmap[] bitmaps = new ImmutableRoaringBitmap[dictionaryIds.length];
    for (int i = 0; i < dictionaryIds.length; i++) {
        bitmaps[i] = invertedIndex.getImmutable(dictionaryIds[i]);
    }
    bitmapBlock = new BitmapBlock(dataSource.getOperatorName(), dataSourceBlock.getMetadata(), startDocId, endDocId, bitmaps, exclusion);
    return bitmapBlock;
}

Example 27 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class BaseOperator method nextBlock.

@Override
public final Block nextBlock() {
    long start = System.currentTimeMillis();
    Block ret = getNextBlock();
    long end = System.currentTimeMillis();
    LOGGER.trace("Time spent in {}: {}", getOperatorName(), (end - start));
    TraceContext.logLatency(getOperatorName(), (end - start));
    return ret;
}

Example 28 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class BaseOperator method nextBlock.

@Override
public final Block nextBlock(BlockId blockId) {
    long start = System.currentTimeMillis();
    Block ret = getNextBlock(blockId);
    long end = System.currentTimeMillis();
    LOGGER.trace("Time spent in {}: {}", getOperatorName(), (end - start));
    TraceContext.logLatency(getOperatorName(), (end - start));
    return ret;
}

Example 29 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class ColumnDataSourceImpl method getNextBlock.

@Override
public Block getNextBlock(BlockId blockId) {
    Block b = null;
    ColumnMetadata columnMetadata = indexContainer.getColumnMetadata();
    if (columnMetadata.isSingleValue()) {
        // TODO: Support sorted index without dictionary.
        if (columnMetadata.hasDictionary() && columnMetadata.isSorted()) {
            b = new SortedSingleValueBlock(blockId, (SortedForwardIndexReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        } else {
            b = new UnSortedSingleValueBlock(blockId, (SingleColumnSingleValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        }
    } else {
        b = new MultiValueBlock(blockId, (SingleColumnMultiValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
    }
    return b;
}

Example 30 with Block

use of com.linkedin.pinot.core.common.Block in project pinot by linkedin.

the class SegmentDumpTool method doMain.

public void doMain(String[] args) throws Exception {
    CmdLineParser parser = new CmdLineParser(this);
    parser.parseArgument(args);
    File segmentDir = new File(segmentPath);
    SegmentMetadata metadata = new SegmentMetadataImpl(segmentDir);
    // All columns by default
    if (columnNames == null) {
        columnNames = new ArrayList<String>(metadata.getSchema().getColumnNames());
        Collections.sort(columnNames);
    }
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDir, ReadMode.mmap);
    Map<String, Dictionary> dictionaries = new HashMap<String, Dictionary>();
    Map<String, BlockSingleValIterator> iterators = new HashMap<String, BlockSingleValIterator>();
    for (String columnName : columnNames) {
        DataSource dataSource = indexSegment.getDataSource(columnName);
        dataSource.open();
        Block block = dataSource.nextBlock();
        BlockValSet blockValSet = block.getBlockValueSet();
        BlockSingleValIterator itr = (BlockSingleValIterator) blockValSet.iterator();
        iterators.put(columnName, itr);
        dictionaries.put(columnName, dataSource.getDictionary());
    }
    System.out.print("Doc\t");
    for (String columnName : columnNames) {
        System.out.print(columnName);
        System.out.print("\t");
    }
    System.out.println();
    for (int i = 0; i < indexSegment.getSegmentMetadata().getTotalDocs(); i++) {
        System.out.print(i);
        System.out.print("\t");
        for (String columnName : columnNames) {
            FieldSpec.DataType columnType = metadata.getSchema().getFieldSpecFor(columnName).getDataType();
            BlockSingleValIterator itr = iterators.get(columnName);
            Integer encodedValue = itr.nextIntVal();
            Object value = dictionaries.get(columnName).get(encodedValue);
            System.out.print(value);
            System.out.print("\t");
        }
        System.out.println();
    }
    if (dumpStarTree) {
        System.out.println();
        File starTreeFile = new File(segmentDir, V1Constants.STAR_TREE_INDEX_FILE);
        StarTreeInterf tree = StarTreeSerDe.fromFile(starTreeFile, ReadMode.mmap);
        tree.printTree();
    }
}