Examples with Dictionary com.linkedin.pinot.core.segment.index.readers.Dictionary used on opensource projects

Example 16 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class SegmentDumpTool method doMain.

public void doMain(String[] args) throws Exception {
    CmdLineParser parser = new CmdLineParser(this);
    parser.parseArgument(args);
    File segmentDir = new File(segmentPath);
    SegmentMetadata metadata = new SegmentMetadataImpl(segmentDir);
    // All columns by default
    if (columnNames == null) {
        columnNames = new ArrayList<String>(metadata.getSchema().getColumnNames());
        Collections.sort(columnNames);
    }
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDir, ReadMode.mmap);
    Map<String, Dictionary> dictionaries = new HashMap<String, Dictionary>();
    Map<String, BlockSingleValIterator> iterators = new HashMap<String, BlockSingleValIterator>();
    for (String columnName : columnNames) {
        DataSource dataSource = indexSegment.getDataSource(columnName);
        dataSource.open();
        Block block = dataSource.nextBlock();
        BlockValSet blockValSet = block.getBlockValueSet();
        BlockSingleValIterator itr = (BlockSingleValIterator) blockValSet.iterator();
        iterators.put(columnName, itr);
        dictionaries.put(columnName, dataSource.getDictionary());
    }
    System.out.print("Doc\t");
    for (String columnName : columnNames) {
        System.out.print(columnName);
        System.out.print("\t");
    }
    System.out.println();
    for (int i = 0; i < indexSegment.getSegmentMetadata().getTotalDocs(); i++) {
        System.out.print(i);
        System.out.print("\t");
        for (String columnName : columnNames) {
            FieldSpec.DataType columnType = metadata.getSchema().getFieldSpecFor(columnName).getDataType();
            BlockSingleValIterator itr = iterators.get(columnName);
            Integer encodedValue = itr.nextIntVal();
            Object value = dictionaries.get(columnName).get(encodedValue);
            System.out.print(value);
            System.out.print("\t");
        }
        System.out.println();
    }
    if (dumpStarTree) {
        System.out.println();
        File starTreeFile = new File(segmentDir, V1Constants.STAR_TREE_INDEX_FILE);
        StarTreeInterf tree = StarTreeSerDe.fromFile(starTreeFile, ReadMode.mmap);
        tree.printTree();
    }
}

Example 17 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class DataFetcher method fetchFloatValues.

/**
   * Fetch the float values for a multi-valued column.
   *
   * @param column column name.
   * @param inDocIds dictionary Id array.
   * @param inStartPos input start position.
   * @param length input length.
   * @param outValues value array buffer.
   * @param outStartPos output start position.
   */
public void fetchFloatValues(String column, int[] inDocIds, int inStartPos, int length, float[][] outValues, int outStartPos) {
    Dictionary dictionary = getDictionaryForColumn(column);
    BlockMultiValIterator iterator = (BlockMultiValIterator) getBlockValIteratorForColumn(column);
    int inEndPos = inStartPos + length;
    for (int i = inStartPos; i < inEndPos; i++, outStartPos++) {
        iterator.skipTo(inDocIds[i]);
        int numValues = iterator.nextIntVal(_reusableMVDictIds);
        outValues[outStartPos] = new float[numValues];
        dictionary.readFloatValues(_reusableMVDictIds, 0, numValues, outValues[outStartPos], 0);
    }
}

Example 18 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class DataFetcher method fetchStringValues.

/**
   *
   * @param column Column for which to fetch the values
   * @param inDocIds Array of docIds for which to fetch the values
   * @param outValues Array of strings where output will be written
   * @param length Length of input docIds
   */
public void fetchStringValues(String column, int[] inDocIds, int inStartPos, int length, String[][] outValues, int outStartPos) {
    Dictionary dictionary = getDictionaryForColumn(column);
    BlockMultiValIterator iterator = (BlockMultiValIterator) getBlockValIteratorForColumn(column);
    int inEndPos = inStartPos + length;
    for (int i = inStartPos; i < inEndPos; i++, outStartPos++) {
        iterator.skipTo(inDocIds[i]);
        int numValues = iterator.nextIntVal(_reusableMVDictIds);
        outValues[outStartPos] = new String[numValues];
        dictionary.readStringValues(_reusableMVDictIds, 0, numValues, outValues[outStartPos], 0);
    }
}

Example 19 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class DataFetcher method fetchDoubleValues.

/**
   * Fetch the values for a single double value column.
   *
   * @param column column name.
   * @param inDocIds dictionary Id array.
   * @param inStartPos input start position.
   * @param length input length.
   * @param outValues value array buffer.
   * @param outStartPos output start position.
   */
public void fetchDoubleValues(String column, int[] inDocIds, int inStartPos, int length, double[] outValues, int outStartPos) {
    Dictionary dictionary = getDictionaryForColumn(column);
    if (dictionary != null) {
        fetchSingleDictIds(column, inDocIds, inStartPos, length, _reusableDictIds, 0);
        dictionary.readDoubleValues(_reusableDictIds, 0, length, outValues, outStartPos);
    } else {
        BlockValSet blockValSet = _columnToBlockValSetMap.get(column);
        blockValSet.getDoubleValues(inDocIds, inStartPos, length, outValues, outStartPos);
    }
}

Example 20 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class DataFetcher method fetchIntValues.

/**
   * Fetch the values for a single int value column.
   *
   * @param column column name.
   * @param inDocIds doc Id array.
   * @param inStartPos input start position.
   * @param length input length.
   * @param outValues value array buffer.
   * @param outStartPos output start position.
   */
public void fetchIntValues(String column, int[] inDocIds, int inStartPos, int length, int[] outValues, int outStartPos) {
    Dictionary dictionary = getDictionaryForColumn(column);
    if (dictionary != null) {
        fetchSingleDictIds(column, inDocIds, inStartPos, length, _reusableDictIds, 0);
        dictionary.readIntValues(_reusableDictIds, 0, length, outValues, outStartPos);
    } else {
        BlockValSet blockValSet = _columnToBlockValSetMap.get(column);
        blockValSet.getIntValues(inDocIds, inStartPos, length, outValues, outStartPos);
    }
}