Examples with Dictionary com.linkedin.pinot.core.segment.index.readers.Dictionary used on opensource projects

Example 21 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class DataFetcher method fetchIntValues.

/**
   * Fetch the int values for a multi-valued column.
   *
   * @param column column name.
   * @param inDocIds dictionary Id array.
   * @param inStartPos input start position.
   * @param length input length.
   * @param outValues value array buffer.
   * @param outStartPos output start position.
   */
public void fetchIntValues(String column, int[] inDocIds, int inStartPos, int length, int[][] outValues, int outStartPos) {
    Dictionary dictionary = getDictionaryForColumn(column);
    BlockMultiValIterator iterator = (BlockMultiValIterator) getBlockValIteratorForColumn(column);
    int inEndPos = inStartPos + length;
    for (int i = inStartPos; i < inEndPos; i++, outStartPos++) {
        iterator.skipTo(inDocIds[i]);
        int numValues = iterator.nextIntVal(_reusableMVDictIds);
        outValues[outStartPos] = new int[numValues];
        dictionary.readIntValues(_reusableMVDictIds, 0, numValues, outValues[outStartPos], 0);
    }
}

Example 22 with Dictionary

use of com.linkedin.pinot.core.segment.index.readers.Dictionary in project pinot by linkedin.

the class BaseSumStarTreeIndexTest method computeSum.

/**
   * Compute 'sum' for a given list of metrics, by scanning the given set of doc-ids.
   *
   * @param segment
   * @param docIdIterator
   * @param metricNames
   * @return
   */
private Map<String, double[]> computeSum(IndexSegment segment, BlockDocIdIterator docIdIterator, List<String> metricNames, List<String> groupByColumns) {
    int docId;
    int numMetrics = metricNames.size();
    Dictionary[] metricDictionaries = new Dictionary[numMetrics];
    BlockSingleValIterator[] metricValIterators = new BlockSingleValIterator[numMetrics];
    int numGroupByColumns = groupByColumns.size();
    Dictionary[] groupByDictionaries = new Dictionary[numGroupByColumns];
    BlockSingleValIterator[] groupByValIterators = new BlockSingleValIterator[numGroupByColumns];
    for (int i = 0; i < numMetrics; i++) {
        String metricName = metricNames.get(i);
        DataSource dataSource = segment.getDataSource(metricName);
        metricDictionaries[i] = dataSource.getDictionary();
        metricValIterators[i] = (BlockSingleValIterator) dataSource.getNextBlock().getBlockValueSet().iterator();
    }
    for (int i = 0; i < numGroupByColumns; i++) {
        String groupByColumn = groupByColumns.get(i);
        DataSource dataSource = segment.getDataSource(groupByColumn);
        groupByDictionaries[i] = dataSource.getDictionary();
        groupByValIterators[i] = (BlockSingleValIterator) dataSource.getNextBlock().getBlockValueSet().iterator();
    }
    Map<String, double[]> result = new HashMap<String, double[]>();
    while ((docId = docIdIterator.next()) != Constants.EOF) {
        StringBuilder stringBuilder = new StringBuilder();
        for (int i = 0; i < numGroupByColumns; i++) {
            groupByValIterators[i].skipTo(docId);
            int dictId = groupByValIterators[i].nextIntVal();
            stringBuilder.append(groupByDictionaries[i].getStringValue(dictId));
            stringBuilder.append("_");
        }
        String key = stringBuilder.toString();
        if (!result.containsKey(key)) {
            result.put(key, new double[numMetrics]);
        }
        double[] sumsSoFar = result.get(key);
        for (int i = 0; i < numMetrics; i++) {
            metricValIterators[i].skipTo(docId);
            int dictId = metricValIterators[i].nextIntVal();
            sumsSoFar[i] += metricDictionaries[i].getDoubleValue(dictId);
        }
    }
    return result;
}