Examples with PinotDataBuffer com.linkedin.pinot.core.segment.memory.PinotDataBuffer used on opensource projects

Example 1 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteSingleColumnMultiValueReaderWriter method addCapacity.

/**
   * This method automatically computes the space needed based on the columnSizeInBytes
   * @param rowCapacity Additional capacity to be added in terms of number of rows
   * @throws RuntimeException
   */
private void addCapacity(int rowCapacity) throws RuntimeException {
    PinotDataBuffer dataBuffer;
    try {
        dataBuffer = PinotDataBuffer.allocateDirect(rowCapacity * columnSizeInBytes);
        //dataBuffer.order(ByteOrder.nativeOrder());
        dataBuffers.add(dataBuffer);
        currentDataWriter = new FixedByteSingleValueMultiColWriter(dataBuffer, rowCapacity, 1, new int[] { columnSizeInBytes });
        dataWriters.add(currentDataWriter);
        FixedByteSingleValueMultiColReader dataFileReader = new FixedByteSingleValueMultiColReader(dataBuffer, rowCapacity, 1, new int[] { columnSizeInBytes });
        dataReaders.add(dataFileReader);
        //update the capacity
        currentCapacity = rowCapacity;
        currentDataWriterIndex = currentDataWriterIndex + 1;
    } catch (Exception e) {
        throw new RuntimeException("Error while expanding the capacity by allocating additional buffer with capacity:" + rowCapacity, e);
    }
}

Example 2 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnIndexContainer method init.

public static ColumnIndexContainer init(SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, IndexLoadingConfigMetadata indexLoadingConfigMetadata) throws IOException {
    String column = metadata.getColumnName();
    boolean loadInverted = false;
    if (indexLoadingConfigMetadata != null) {
        if (indexLoadingConfigMetadata.getLoadingInvertedIndexColumns() != null) {
            loadInverted = indexLoadingConfigMetadata.getLoadingInvertedIndexColumns().contains(metadata.getColumnName());
        }
    }
    ImmutableDictionaryReader dictionary = null;
    if (metadata.hasDictionary()) {
        PinotDataBuffer dictionaryBuffer = segmentReader.getIndexFor(column, ColumnIndexType.DICTIONARY);
        dictionary = load(metadata, dictionaryBuffer);
    }
    // TODO: Support sorted index without dictionary.
    if (dictionary != null && metadata.isSorted() && metadata.isSingleValue()) {
        return loadSorted(column, segmentReader, metadata, dictionary);
    }
    if (metadata.isSingleValue()) {
        return loadUnsorted(column, segmentReader, metadata, dictionary, loadInverted);
    }
    return loadMultiValue(column, segmentReader, metadata, dictionary, loadInverted);
}

Example 3 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnIndexContainer method loadUnsorted.

private static ColumnIndexContainer loadUnsorted(String column, SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, ImmutableDictionaryReader dictionary, boolean loadInverted) throws IOException {
    PinotDataBuffer fwdIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.FORWARD_INDEX);
    SingleColumnSingleValueReader fwdIndexReader;
    if (dictionary != null) {
        fwdIndexReader = new FixedBitSingleValueReader(fwdIndexBuffer, metadata.getTotalDocs(), metadata.getBitsPerElement(), metadata.hasNulls());
    } else {
        // TODO: Replace hard-coded compressor with getting information from meta-data.
        fwdIndexReader = getRawIndexReader(fwdIndexBuffer, metadata.getDataType());
    }
    BitmapInvertedIndexReader invertedIndex = null;
    if (loadInverted) {
        PinotDataBuffer invertedIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.INVERTED_INDEX);
        invertedIndex = new BitmapInvertedIndexReader(invertedIndexBuffer, metadata.getCardinality());
    }
    return new UnsortedSVColumnIndexContainer(column, metadata, fwdIndexReader, dictionary, invertedIndex);
}

Example 4 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnIndexContainer method loadMultiValue.

private static ColumnIndexContainer loadMultiValue(String column, SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, ImmutableDictionaryReader dictionary, boolean loadInverted) throws IOException {
    PinotDataBuffer fwdIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.FORWARD_INDEX);
    SingleColumnMultiValueReader<? extends ReaderContext> fwdIndexReader = new FixedBitMultiValueReader(fwdIndexBuffer, metadata.getTotalDocs(), metadata.getTotalNumberOfEntries(), metadata.getBitsPerElement(), false);
    BitmapInvertedIndexReader invertedIndex = null;
    if (loadInverted) {
        PinotDataBuffer invertedIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.INVERTED_INDEX);
        invertedIndex = new BitmapInvertedIndexReader(invertedIndexBuffer, metadata.getCardinality());
    }
    return new UnSortedMVColumnIndexContainer(column, metadata, fwdIndexReader, dictionary, invertedIndex);
}

Example 5 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnMinMaxValueGenerator method addColumnMinMaxValueForColumn.

private void addColumnMinMaxValueForColumn(String columnName) throws Exception {
    // Skip column without dictionary or with min/max value already set
    ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(columnName);
    if ((!columnMetadata.hasDictionary()) || (columnMetadata.getMinValue() != null)) {
        return;
    }
    PinotDataBuffer dictionaryBuffer = _segmentWriter.getIndexFor(columnName, ColumnIndexType.DICTIONARY);
    FieldSpec.DataType dataType = columnMetadata.getDataType();
    switch(dataType) {
        case INT:
            IntDictionary intDictionary = new IntDictionary(dictionaryBuffer, columnMetadata);
            SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(_segmentProperties, columnName, intDictionary.getStringValue(0), intDictionary.getStringValue(intDictionary.length() - 1));
            break;
        case LONG:
            LongDictionary longDictionary = new LongDictionary(dictionaryBuffer, columnMetadata);
            SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(_segmentProperties, columnName, longDictionary.getStringValue(0), longDictionary.getStringValue(longDictionary.length() - 1));
            break;
        case FLOAT:
            FloatDictionary floatDictionary = new FloatDictionary(dictionaryBuffer, columnMetadata);
            SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(_segmentProperties, columnName, floatDictionary.getStringValue(0), floatDictionary.getStringValue(floatDictionary.length() - 1));
            break;
        case DOUBLE:
            DoubleDictionary doubleDictionary = new DoubleDictionary(dictionaryBuffer, columnMetadata);
            SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(_segmentProperties, columnName, doubleDictionary.getStringValue(0), doubleDictionary.getStringValue(doubleDictionary.length() - 1));
            break;
        case STRING:
            StringDictionary stringDictionary = new StringDictionary(dictionaryBuffer, columnMetadata);
            SegmentColumnarIndexCreator.addColumnMinMaxValueInfo(_segmentProperties, columnName, stringDictionary.get(0), stringDictionary.get(stringDictionary.length() - 1));
            break;
        default:
            throw new IllegalStateException("Unsupported data type: " + dataType + " for column: " + columnName);
    }
    _minMaxValueAdded = true;
}