Examples with FixedBitSingleValueReader com.linkedin.pinot.core.io.reader.impl.v2.FixedBitSingleValueReader used on opensource projects

Example 1 with FixedBitSingleValueReader

use of com.linkedin.pinot.core.io.reader.impl.v2.FixedBitSingleValueReader in project pinot by linkedin.

the class ColumnIndexContainer method loadUnsorted.

private static ColumnIndexContainer loadUnsorted(String column, SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, ImmutableDictionaryReader dictionary, boolean loadInverted) throws IOException {
    PinotDataBuffer fwdIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.FORWARD_INDEX);
    SingleColumnSingleValueReader fwdIndexReader;
    if (dictionary != null) {
        fwdIndexReader = new FixedBitSingleValueReader(fwdIndexBuffer, metadata.getTotalDocs(), metadata.getBitsPerElement(), metadata.hasNulls());
    } else {
        // TODO: Replace hard-coded compressor with getting information from meta-data.
        fwdIndexReader = getRawIndexReader(fwdIndexBuffer, metadata.getDataType());
    }
    BitmapInvertedIndexReader invertedIndex = null;
    if (loadInverted) {
        PinotDataBuffer invertedIndexBuffer = segmentReader.getIndexFor(column, ColumnIndexType.INVERTED_INDEX);
        invertedIndex = new BitmapInvertedIndexReader(invertedIndexBuffer, metadata.getCardinality());
    }
    return new UnsortedSVColumnIndexContainer(column, metadata, fwdIndexReader, dictionary, invertedIndex);
}

Example 2 with FixedBitSingleValueReader

use of com.linkedin.pinot.core.io.reader.impl.v2.FixedBitSingleValueReader in project pinot by linkedin.

the class InvertedIndexHandler method createInvertedIndexForColumn.

private void createInvertedIndexForColumn(ColumnMetadata columnMetadata) throws IOException {
    String column = columnMetadata.getColumnName();
    File inProgress = new File(indexDir, column + ".inv.inprogress");
    File invertedIndexFile = new File(indexDir, column + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    if (!inProgress.exists()) {
        if (segmentWriter.hasIndexFor(column, ColumnIndexType.INVERTED_INDEX)) {
            // Skip creating inverted index if already exists.
            LOGGER.info("Found inverted index for segment: {}, column: {}", segmentName, column);
            return;
        }
        // Create a marker file.
        FileUtils.touch(inProgress);
    } else {
        // Marker file exists, which means last run gets interrupted.
        // Remove inverted index if exists.
        // For v1 and v2, it's the actual inverted index. For v3, it's the temporary inverted index.
        FileUtils.deleteQuietly(invertedIndexFile);
    }
    // Create new inverted index for the column.
    LOGGER.info("Creating new inverted index for segment: {}, column: {}", segmentName, column);
    int totalDocs = columnMetadata.getTotalDocs();
    OffHeapBitmapInvertedIndexCreator creator = new OffHeapBitmapInvertedIndexCreator(indexDir, columnMetadata.getCardinality(), totalDocs, columnMetadata.getTotalNumberOfEntries(), columnMetadata.getFieldSpec());
    try (DataFileReader fwdIndex = getForwardIndexReader(columnMetadata, segmentWriter)) {
        if (columnMetadata.isSingleValue()) {
            // Single-value column.
            FixedBitSingleValueReader svFwdIndex = (FixedBitSingleValueReader) fwdIndex;
            for (int i = 0; i < totalDocs; i++) {
                creator.add(i, svFwdIndex.getInt(i));
            }
        } else {
            // Multi-value column.
            SingleColumnMultiValueReader mvFwdIndex = (SingleColumnMultiValueReader) fwdIndex;
            int[] dictIds = new int[columnMetadata.getMaxNumberOfMultiValues()];
            for (int i = 0; i < totalDocs; i++) {
                int len = mvFwdIndex.getIntArray(i, dictIds);
                creator.add(i, dictIds, len);
            }
        }
    }
    creator.seal();
    // For v3, write the generated inverted index file into the single file and remove it.
    if (segmentVersion == SegmentVersion.v3) {
        LoaderUtils.writeIndexToV3Format(segmentWriter, column, invertedIndexFile, ColumnIndexType.INVERTED_INDEX);
    }
    // Delete the marker file.
    FileUtils.deleteQuietly(inProgress);
    LOGGER.info("Created inverted index for segment: {}, column: {}", segmentName, column);
}

Example 3 with FixedBitSingleValueReader

use of com.linkedin.pinot.core.io.reader.impl.v2.FixedBitSingleValueReader in project pinot by linkedin.

the class BenchmarkFileRead method readSVs.

/*@Benchmark
  @OutputTimeUnit(TimeUnit.MILLISECONDS)
  public void test() {
    byteBuffer.rewind();
    byte[] rawData = new byte[length];

    byteBuffer.rewind();
    for (int i = 0; i < length; i++) {
      rawData[i] = byteBuffer.get();
    }
  }*/
@Benchmark
@BenchmarkMode({ Mode.SampleTime })
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public void readSVs() throws IOException {
    int rows = 25000000;
    int columnSizeInBits = 3;
    boolean isMMap = true;
    boolean hasNulls = false;
    PinotDataBuffer dataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "benchmark");
    FixedBitSingleValueReader reader = new FixedBitSingleValueReader(dataBuffer, rows, columnSizeInBits, hasNulls);
    int[] result2 = new int[rows];
    for (int i = 0; i < rows; i++) {
        result2[i] = reader.getInt(i);
    }
}

Example 4 with FixedBitSingleValueReader

use of com.linkedin.pinot.core.io.reader.impl.v2.FixedBitSingleValueReader in project pinot by linkedin.

the class FixedBitSingleValueTest method testV2.

@Test
public void testV2() throws Exception {
    int ROWS = 1000;
    for (int numBits = 1; numBits < 32; numBits++) {
        File file = new File(this.getClass().getName() + "_" + numBits + ".test");
        FixedBitSingleValueWriter writer = new FixedBitSingleValueWriter(file, ROWS, numBits);
        int[] data = new int[ROWS];
        Random random = new Random();
        int max = (int) Math.pow(2, numBits);
        for (int i = 0; i < ROWS; i++) {
            data[i] = random.nextInt(max);
            writer.setInt(i, data[i]);
        }
        writer.close();
        PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
        FixedBitSingleValueReader reader = new FixedBitSingleValueReader(heapBuffer, ROWS, numBits);
        int[] read = new int[ROWS];
        for (int i = 0; i < ROWS; i++) {
            read[i] = reader.getInt(i);
        //Assert.assertEquals(reader.getInt(i), data[i],
        //  "Failed for bit:" + numBits + " Expected " + data[i] + " but found " + reader.getInt(i) + "  at " + i);
        }
        LOGGER.trace(Arrays.toString(data));
        LOGGER.trace(Arrays.toString(read));
        reader.close();
        heapBuffer.close();
        file.delete();
    }
}