Examples with SingleColumnSingleValueReader com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader used on opensource projects

Example 1 with SingleColumnSingleValueReader

use of com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader in project pinot by linkedin.

the class ColumnIndexContainer method loadUnsorted.

private static ColumnIndexContainer loadUnsorted(String column, SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, ImmutableDictionaryReader dictionary, boolean loadInverted) throws IOException {
    PinotDataBuffer fwdIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.FORWARD_INDEX);
    SingleColumnSingleValueReader fwdIndexReader;
    if (dictionary != null) {
        fwdIndexReader = new FixedBitSingleValueReader(fwdIndexBuffer, metadata.getTotalDocs(), metadata.getBitsPerElement(), metadata.hasNulls());
    } else {
        // TODO: Replace hard-coded compressor with getting information from meta-data.
        fwdIndexReader = getRawIndexReader(fwdIndexBuffer, metadata.getDataType());
    }
    BitmapInvertedIndexReader invertedIndex = null;
    if (loadInverted) {
        PinotDataBuffer invertedIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.INVERTED_INDEX);
        invertedIndex = new BitmapInvertedIndexReader(invertedIndexBuffer, metadata.getCardinality());
    }
    return new UnsortedSVColumnIndexContainer(column, metadata, fwdIndexReader, dictionary, invertedIndex);
}

Example 2 with SingleColumnSingleValueReader

use of com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader in project pinot by linkedin.

the class ColumnIndexContainer method getRawIndexReader.

public static SingleColumnSingleValueReader getRawIndexReader(PinotDataBuffer fwdIndexBuffer, FieldSpec.DataType dataType) throws IOException {
    SingleColumnSingleValueReader reader;
    // TODO: Make compression/decompression configurable.
    ChunkDecompressor decompressor = ChunkCompressorFactory.getDecompressor("snappy");
    switch(dataType) {
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
            reader = new FixedByteChunkSingleValueReader(fwdIndexBuffer, decompressor);
            break;
        case STRING:
            reader = new VarByteChunkSingleValueReader(fwdIndexBuffer, decompressor);
            break;
        default:
            throw new IllegalArgumentException("Illegal data type for raw index reader: " + dataType);
    }
    return reader;
}

Example 3 with SingleColumnSingleValueReader

use of com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader in project pinot by linkedin.

the class SegmentFormatConverterV1ToV2 method convert.

@Override
public void convert(File indexSegmentDir) throws Exception {
    SegmentMetadataImpl segmentMetadataImpl = new SegmentMetadataImpl(indexSegmentDir);
    SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegmentDir, segmentMetadataImpl, ReadMode.mmap);
    Set<String> columns = segmentMetadataImpl.getAllColumns();
    SegmentDirectory.Writer segmentWriter = segmentDirectory.createWriter();
    for (String column : columns) {
        ColumnMetadata columnMetadata = segmentMetadataImpl.getColumnMetadataFor(column);
        if (columnMetadata.isSorted()) {
            // no need to change sorted forward index
            continue;
        }
        PinotDataBuffer fwdIndexBuffer = segmentWriter.getIndexFor(column, ColumnIndexType.FORWARD_INDEX);
        if (columnMetadata.isSingleValue() && !columnMetadata.isSorted()) {
            // since we use dictionary to encode values, we wont have any negative values in forward
            // index
            boolean signed = false;
            SingleColumnSingleValueReader v1Reader = new com.linkedin.pinot.core.io.reader.impl.v1.FixedBitSingleValueReader(fwdIndexBuffer, segmentMetadataImpl.getTotalDocs(), columnMetadata.getBitsPerElement(), false);
            File convertedFwdIndexFile = new File(indexSegmentDir, column + V1Constants.Indexes.UN_SORTED_SV_FWD_IDX_FILE_EXTENTION + ".tmp");
            SingleColumnSingleValueWriter v2Writer = new com.linkedin.pinot.core.io.writer.impl.v2.FixedBitSingleValueWriter(convertedFwdIndexFile, segmentMetadataImpl.getTotalDocs(), columnMetadata.getBitsPerElement());
            for (int row = 0; row < segmentMetadataImpl.getTotalDocs(); row++) {
                int value = v1Reader.getInt(row);
                v2Writer.setInt(row, value);
            }
            v1Reader.close();
            v2Writer.close();
            File fwdIndexFileCopy = new File(indexSegmentDir, column + V1Constants.Indexes.UN_SORTED_SV_FWD_IDX_FILE_EXTENTION + ".orig");
            segmentWriter.removeIndex(column, ColumnIndexType.FORWARD_INDEX);
            // FIXME
            PinotDataBuffer newIndexBuffer = segmentWriter.newIndexFor(column, ColumnIndexType.FORWARD_INDEX, (int) convertedFwdIndexFile.length());
            newIndexBuffer.readFrom(convertedFwdIndexFile);
            convertedFwdIndexFile.delete();
        }
        if (!columnMetadata.isSingleValue()) {
            // since we use dictionary to encode values, we wont have any negative values in forward
            // index
            boolean signed = false;
            SingleColumnMultiValueReader v1Reader = new com.linkedin.pinot.core.io.reader.impl.v1.FixedBitMultiValueReader(fwdIndexBuffer, segmentMetadataImpl.getTotalDocs(), columnMetadata.getTotalNumberOfEntries(), columnMetadata.getBitsPerElement(), signed);
            File convertedFwdIndexFile = new File(indexSegmentDir, column + V1Constants.Indexes.UN_SORTED_MV_FWD_IDX_FILE_EXTENTION + ".tmp");
            SingleColumnMultiValueWriter v2Writer = new com.linkedin.pinot.core.io.writer.impl.v2.FixedBitMultiValueWriter(convertedFwdIndexFile, segmentMetadataImpl.getTotalDocs(), columnMetadata.getTotalNumberOfEntries(), columnMetadata.getBitsPerElement());
            int[] values = new int[columnMetadata.getMaxNumberOfMultiValues()];
            for (int row = 0; row < segmentMetadataImpl.getTotalDocs(); row++) {
                int length = v1Reader.getIntArray(row, values);
                int[] copy = new int[length];
                System.arraycopy(values, 0, copy, 0, length);
                v2Writer.setIntArray(row, copy);
            }
            v1Reader.close();
            v2Writer.close();
            segmentWriter.removeIndex(column, ColumnIndexType.FORWARD_INDEX);
            PinotDataBuffer newIndexBuffer = segmentWriter.newIndexFor(column, ColumnIndexType.FORWARD_INDEX, (int) convertedFwdIndexFile.length());
            newIndexBuffer.readFrom(convertedFwdIndexFile);
            convertedFwdIndexFile.delete();
        }
    }
    File metadataFile = new File(indexSegmentDir, V1Constants.MetadataKeys.METADATA_FILE_NAME);
    File metadataFileCopy = new File(indexSegmentDir, V1Constants.MetadataKeys.METADATA_FILE_NAME + ".orig");
    bis = new BufferedInputStream(new FileInputStream(metadataFile));
    bos = new BufferedOutputStream(new FileOutputStream(metadataFileCopy));
    IOUtils.copy(bis, bos);
    bis.close();
    bos.close();
    final PropertiesConfiguration properties = new PropertiesConfiguration(metadataFileCopy);
    // update the segment version
    properties.setProperty(V1Constants.MetadataKeys.Segment.SEGMENT_VERSION, SegmentVersion.v2.toString());
    metadataFile.delete();
    properties.save(metadataFile);
}

Example 4 with SingleColumnSingleValueReader

use of com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader in project pinot by linkedin.

the class ColumnDataSourceImpl method getNextBlock.

@Override
public Block getNextBlock(BlockId blockId) {
    Block b = null;
    ColumnMetadata columnMetadata = indexContainer.getColumnMetadata();
    if (columnMetadata.isSingleValue()) {
        // TODO: Support sorted index without dictionary.
        if (columnMetadata.hasDictionary() && columnMetadata.isSorted()) {
            b = new SortedSingleValueBlock(blockId, (SortedForwardIndexReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        } else {
            b = new UnSortedSingleValueBlock(blockId, (SingleColumnSingleValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        }
    } else {
        b = new MultiValueBlock(blockId, (SingleColumnMultiValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
    }
    return b;
}

Example 5 with SingleColumnSingleValueReader

use of com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader in project pinot by linkedin.

the class IntArraysTest method test1.

@Test
public void test1() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final Map<String, ColumnMetadata> metadataMap = ((SegmentMetadataImpl) heapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
        final DataFileReader heapArray = heapSegment.getForwardIndexReaderFor(column);
        final DataFileReader mmapArray = mmapSegment.getForwardIndexReaderFor(column);
        if (metadataMap.get(column).isSingleValue()) {
            final SingleColumnSingleValueReader svHeapReader = (SingleColumnSingleValueReader) heapArray;
            final SingleColumnSingleValueReader mvMmapReader = (SingleColumnSingleValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                Assert.assertEquals(mvMmapReader.getInt(i), svHeapReader.getInt(i));
            }
        } else {
            final SingleColumnMultiValueReader svHeapReader = (SingleColumnMultiValueReader) heapArray;
            final SingleColumnMultiValueReader mvMmapReader = (SingleColumnMultiValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                final int[] i_1 = new int[1000];
                final int[] j_i = new int[1000];
                Assert.assertEquals(mvMmapReader.getIntArray(i, j_i), svHeapReader.getIntArray(i, i_1));
            }
        }
    }
}