Examples with ImmutableDictionaryReader com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader used on opensource projects

Example 1 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class IndexSegmentImpl method destroy.

@Override
public void destroy() {
    LOGGER.info("Trying to destroy segment : {}", this.getSegmentName());
    for (String column : indexContainerMap.keySet()) {
        ColumnIndexContainer columnIndexContainer = indexContainerMap.get(column);
        try {
            if (columnIndexContainer.hasDictionary()) {
                ImmutableDictionaryReader dictionary = columnIndexContainer.getDictionary();
                dictionary.close();
            }
        } catch (Exception e) {
            LOGGER.error("Error when close dictionary index for column : " + column, e);
        }
        try {
            columnIndexContainer.getForwardIndex().close();
        } catch (Exception e) {
            LOGGER.error("Error when close forward index for column : " + column, e);
        }
        try {
            if (columnIndexContainer.getInvertedIndex() != null) {
                columnIndexContainer.getInvertedIndex().close();
            }
        } catch (Exception e) {
            LOGGER.error("Error when close inverted index for column : " + column, e);
        }
    }
    try {
        segmentDirectory.close();
    } catch (Exception e) {
        LOGGER.error("Failed to close segment directory: {}. Continuing with error.", segmentDirectory, e);
    }
    indexContainerMap.clear();
}

Example 2 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class ColumnIndexContainer method init.

public static ColumnIndexContainer init(SegmentDirectory.Reader segmentReader, ColumnMetadata metadata, IndexLoadingConfigMetadata indexLoadingConfigMetadata) throws IOException {
    String column = metadata.getColumnName();
    boolean loadInverted = false;
    if (indexLoadingConfigMetadata != null) {
        if (indexLoadingConfigMetadata.getLoadingInvertedIndexColumns() != null) {
            loadInverted = indexLoadingConfigMetadata.getLoadingInvertedIndexColumns().contains(metadata.getColumnName());
        }
    }
    ImmutableDictionaryReader dictionary = null;
    if (metadata.hasDictionary()) {
        PinotDataBuffer dictionaryBuffer = segmentReader.getIndexFor(column, ColumnIndexType.DICTIONARY);
        dictionary = load(metadata, dictionaryBuffer);
    }
    // TODO: Support sorted index without dictionary.
    if (dictionary != null && metadata.isSorted() && metadata.isSingleValue()) {
        return loadSorted(column, segmentReader, metadata, dictionary);
    }
    if (metadata.isSingleValue()) {
        return loadUnsorted(column, segmentReader, metadata, dictionary, loadInverted);
    }
    return loadMultiValue(column, segmentReader, metadata, dictionary, loadInverted);
}

Example 3 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test4.

@Test(enabled = false)
public void test4() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final ImmutableDictionaryReader d = segment.getDictionaryFor("column1");
    final List<String> rhs = new ArrayList<String>();
    rhs.add(d.get(new Random().nextInt(d.length())).toString());
    final Predicate p = new EqPredicate("column1", rhs);
    final DataSource ds = segment.getDataSource("column1", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}

Example 4 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class DictionariesTest method test2.

@Test
public void test2() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.mmap);
    final Map<String, ColumnMetadata> metadataMap = ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
        final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
        final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
        final Set<Object> uniques = uniqueEntries.get(column);
        final List<Object> list = Arrays.asList(uniques.toArray());
        Collections.shuffle(list);
        for (final Object entry : list) {
            Assert.assertEquals(mmapDictionary.indexOf(entry), heapDictionary.indexOf(entry));
            if (!column.equals("pageKey")) {
                Assert.assertFalse(heapDictionary.indexOf(entry) < 0);
                Assert.assertFalse(mmapDictionary.indexOf(entry) < 0);
            }
        }
    }
}

Example 5 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class BitmapPerformanceBenchmark method iterationSpeed.

public static void iterationSpeed(String indexSegmentDir, String column) throws Exception {
    File indexSegment = new File(indexSegmentDir);
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexSegment);
    Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
    Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
    Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
    File bitMapIndexFile = new File(indexSegmentDir, column + ".bitmap.inv");
    ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
    int cardinality = columnMetadata.getCardinality();
    cardinalityMap.put(column, cardinality);
    PinotDataBuffer bitMapDataBuffer = PinotDataBuffer.fromFile(bitMapIndexFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
    BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitMapDataBuffer, cardinality);
    File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
    SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegment, segmentMetadata, ReadMode.mmap);
    SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
    ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
    ImmutableDictionaryReader dictionary = container.getDictionary();
    dictionaryMap.put(column, dictionary);
    // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
    bitMapIndexMap.put(column, bitmapInvertedIndex);
    int dictId = dictionary.indexOf("na.us");
    ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
    Iterator<Integer> iterator = immutable.iterator();
    int count = 0;
    long start = System.currentTimeMillis();
    while (iterator.hasNext()) {
        iterator.next();
        count = count + 1;
    }
    long end = System.currentTimeMillis();
    System.out.println(" matched: " + count + " Time to iterate:" + (end - start));
    bitMapDataBuffer.close();
}