Examples with DataFileReader com.linkedin.pinot.core.io.reader.DataFileReader used on opensource projects

Example 1 with DataFileReader

use of com.linkedin.pinot.core.io.reader.DataFileReader in project pinot by linkedin.

the class RealtimeSegmentImpl method destroy.

@Override
public void destroy() {
    LOGGER.info("Trying to close RealtimeSegmentImpl : {}", this.getSegmentName());
    for (DataFileReader dfReader : columnIndexReaderWriterMap.values()) {
        try {
            dfReader.close();
        } catch (IOException e) {
            LOGGER.error("Failed to close index. Service will continue with potential memory leak, error: ", e);
        // fall through to close other segments
        }
    }
    // clear map now that index is closed to prevent accidental usage
    columnIndexReaderWriterMap.clear();
    for (RealtimeInvertedIndex index : invertedIndexMap.values()) {
        try {
            index.close();
        } catch (IOException e) {
            LOGGER.error("Failed to close inverted index. Service will continue with memory leaks, error: ", e);
        }
    }
    invertedIndexMap.clear();
    _segmentMetadata.close();
}

Example 2 with DataFileReader

use of com.linkedin.pinot.core.io.reader.DataFileReader in project pinot by linkedin.

the class InvertedIndexHandler method createInvertedIndexForColumn.

private void createInvertedIndexForColumn(ColumnMetadata columnMetadata) throws IOException {
    String column = columnMetadata.getColumnName();
    File inProgress = new File(indexDir, column + ".inv.inprogress");
    File invertedIndexFile = new File(indexDir, column + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    if (!inProgress.exists()) {
        if (segmentWriter.hasIndexFor(column, ColumnIndexType.INVERTED_INDEX)) {
            // Skip creating inverted index if already exists.
            LOGGER.info("Found inverted index for segment: {}, column: {}", segmentName, column);
            return;
        }
        // Create a marker file.
        FileUtils.touch(inProgress);
    } else {
        // Marker file exists, which means last run gets interrupted.
        // Remove inverted index if exists.
        // For v1 and v2, it's the actual inverted index. For v3, it's the temporary inverted index.
        FileUtils.deleteQuietly(invertedIndexFile);
    }
    // Create new inverted index for the column.
    LOGGER.info("Creating new inverted index for segment: {}, column: {}", segmentName, column);
    int totalDocs = columnMetadata.getTotalDocs();
    OffHeapBitmapInvertedIndexCreator creator = new OffHeapBitmapInvertedIndexCreator(indexDir, columnMetadata.getCardinality(), totalDocs, columnMetadata.getTotalNumberOfEntries(), columnMetadata.getFieldSpec());
    try (DataFileReader fwdIndex = getForwardIndexReader(columnMetadata, segmentWriter)) {
        if (columnMetadata.isSingleValue()) {
            // Single-value column.
            FixedBitSingleValueReader svFwdIndex = (FixedBitSingleValueReader) fwdIndex;
            for (int i = 0; i < totalDocs; i++) {
                creator.add(i, svFwdIndex.getInt(i));
            }
        } else {
            // Multi-value column.
            SingleColumnMultiValueReader mvFwdIndex = (SingleColumnMultiValueReader) fwdIndex;
            int[] dictIds = new int[columnMetadata.getMaxNumberOfMultiValues()];
            for (int i = 0; i < totalDocs; i++) {
                int len = mvFwdIndex.getIntArray(i, dictIds);
                creator.add(i, dictIds, len);
            }
        }
    }
    creator.seal();
    // For v3, write the generated inverted index file into the single file and remove it.
    if (segmentVersion == SegmentVersion.v3) {
        LoaderUtils.writeIndexToV3Format(segmentWriter, column, invertedIndexFile, ColumnIndexType.INVERTED_INDEX);
    }
    // Delete the marker file.
    FileUtils.deleteQuietly(inProgress);
    LOGGER.info("Created inverted index for segment: {}, column: {}", segmentName, column);
}

Example 3 with DataFileReader

use of com.linkedin.pinot.core.io.reader.DataFileReader in project pinot by linkedin.

the class IntArraysTest method test1.

@Test
public void test1() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final Map<String, ColumnMetadata> metadataMap = ((SegmentMetadataImpl) heapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
        final DataFileReader heapArray = heapSegment.getForwardIndexReaderFor(column);
        final DataFileReader mmapArray = mmapSegment.getForwardIndexReaderFor(column);
        if (metadataMap.get(column).isSingleValue()) {
            final SingleColumnSingleValueReader svHeapReader = (SingleColumnSingleValueReader) heapArray;
            final SingleColumnSingleValueReader mvMmapReader = (SingleColumnSingleValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                Assert.assertEquals(mvMmapReader.getInt(i), svHeapReader.getInt(i));
            }
        } else {
            final SingleColumnMultiValueReader svHeapReader = (SingleColumnMultiValueReader) heapArray;
            final SingleColumnMultiValueReader mvMmapReader = (SingleColumnMultiValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                final int[] i_1 = new int[1000];
                final int[] j_i = new int[1000];
                Assert.assertEquals(mvMmapReader.getIntArray(i, j_i), svHeapReader.getIntArray(i, i_1));
            }
        }
    }
}