Examples with ImmutableDictionaryReader com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader used on opensource projects

Example 6 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class BitmapPerformanceBenchmark method benchmarkIntersetionAndUnion.

public static void benchmarkIntersetionAndUnion(String indexSegmentDir) throws ConfigurationException, IOException, Exception {
    File[] listFiles = new File(indexSegmentDir).listFiles();
    File indexDir = new File(indexSegmentDir);
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
    Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
    Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
    Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
    for (File file : listFiles) {
        if (!file.getName().endsWith("bitmap.inv")) {
            continue;
        }
        String column = file.getName().replaceAll(".bitmap.inv", "");
        ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
        int cardinality = columnMetadata.getCardinality();
        cardinalityMap.put(column, cardinality);
        System.out.println(column + "\t\t\t" + cardinality + "  \t" + columnMetadata.getDataType());
        PinotDataBuffer bitmapDataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
        BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitmapDataBuffer, cardinality);
        File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
        SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, segmentMetadata, ReadMode.mmap);
        SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
        ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
        ImmutableDictionaryReader dictionary = container.getDictionary();
        if (columnMetadata.getDataType() == DataType.INT) {
            System.out.println("BitmapPerformanceBenchmark.main()");
            assert dictionary instanceof IntDictionary;
        }
        dictionaryMap.put(column, dictionary);
        // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
        bitMapIndexMap.put(column, bitmapInvertedIndex);
        bitmapDataBuffer.close();
    }
    List<String> dimensionNamesList = segmentMetadata.getSchema().getDimensionNames();
    Collections.shuffle(dimensionNamesList);
    int NUM_TEST = 100;
    final int MAX_DIMENSIONS_PER_DIMENSION = 1;
    int MAX_DIMENSIONS_IN_WHERE_CLAUSE = 3;
    Random random = new Random();
    for (int numDimensions = 1; numDimensions <= MAX_DIMENSIONS_IN_WHERE_CLAUSE; numDimensions++) {
        for (int numValuesPerDimension = 1; numValuesPerDimension <= MAX_DIMENSIONS_PER_DIMENSION; numValuesPerDimension++) {
            int runCount = 0;
            while (runCount < NUM_TEST) {
                Collections.shuffle(dimensionNamesList);
                List<ImmutableRoaringBitmap> bitMaps = new ArrayList<ImmutableRoaringBitmap>();
                List<String> columnNameValuePairs = new ArrayList<String>();
                for (int i = 0; i < numDimensions; i++) {
                    String columnName = dimensionNamesList.get(i);
                    InvertedIndexReader bitmapInvertedIndex = bitMapIndexMap.get(columnName);
                    for (int j = 0; j < numValuesPerDimension; j++) {
                        int dictId = random.nextInt(cardinalityMap.get(columnName));
                        String dictValue = dictionaryMap.get(columnName).getStringValue(dictId);
                        columnNameValuePairs.add(columnName + ":" + dictValue);
                        ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
                        bitMaps.add(immutable);
                    }
                }
                System.out.println("START**********************************");
                int[] cardinality = new int[bitMaps.size()];
                int[] sizes = new int[bitMaps.size()];
                for (int i = 0; i < bitMaps.size(); i++) {
                    ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i);
                    cardinality[i] = immutableRoaringBitmap.getCardinality();
                    sizes[i] = immutableRoaringBitmap.getSizeInBytes();
                }
                System.out.println("\t#bitmaps:" + bitMaps.size());
                System.out.println("\tinput values:" + columnNameValuePairs);
                System.out.println("\tinput cardinality:" + Arrays.toString(cardinality));
                System.out.println("\tinput sizes:" + Arrays.toString(sizes));
                and(bitMaps);
                or(bitMaps);
                System.out.println("END**********************************");
                runCount = runCount + 1;
            }
        }
    }
}

Example 7 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class DictionaryDumper method main.

public static void main(String[] args) throws Exception {
    if (args.length != 3) {
        LOGGER.error("Usage: DictionaryDumper <segmentDirectory> <dimensionName> <comma-separated dictionaryIds>");
        System.exit(1);
    }
    File[] segmentDirs = new File(args[0]).listFiles();
    for (int i = 0; i < segmentDirs.length; i++) {
        File indexSegmentDir = segmentDirs[i];
        System.out.println("Loading " + indexSegmentDir.getName());
        IndexSegmentImpl indexSegmentImpl = (IndexSegmentImpl) Loaders.IndexSegment.load(indexSegmentDir, ReadMode.heap);
        ImmutableDictionaryReader colDictionary = indexSegmentImpl.getDictionaryFor(args[1]);
        List<String> strIdList = Arrays.asList(args[2].split(","));
        for (String strId : strIdList) {
            int id = Integer.valueOf(strId);
            String s = colDictionary.getStringValue(id);
            System.out.println(String.format("%d -> %s", id, s));
        }
    }
}

Example 8 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class StringDictionaryPerfTest method perfTestLookups.

/**
   * Measures the performance of string dictionary lookups by performing the provided
   * number of lookups to random indices.
   *
   * @param numLookups Number of lookups to perform
   * @throws Exception
   */
public void perfTestLookups(int numLookups) throws Exception {
    IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(_indexDir, ReadMode.heap);
    ImmutableDictionaryReader dictionary = segment.getDictionaryFor(COLUMN_NAME);
    Random random = new Random(System.nanoTime());
    long start = System.currentTimeMillis();
    for (int i = 0; i < numLookups; i++) {
        int index = 1 + random.nextInt(_dictLength);
        dictionary.indexOf(_inputStrings[index]);
    }
    FileUtils.deleteQuietly(_indexDir);
    System.out.println("Total time for " + TOTAL_NUM_LOOKUPS + " lookups: " + (System.currentTimeMillis() - start));
}

Example 9 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class RangeOfflineDictionaryPredicateEvaluatorTest method createReader.

private ImmutableDictionaryReader createReader(int rangeStart, int rangeEnd) {
    ImmutableDictionaryReader reader = mock(ImmutableDictionaryReader.class);
    when(reader.indexOf("lower")).thenReturn(rangeStart);
    when(reader.indexOf("upper")).thenReturn(rangeEnd);
    when(reader.length()).thenReturn(DICT_LEN);
    return reader;
}

Example 10 with ImmutableDictionaryReader

use of com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader in project pinot by linkedin.

the class DictionariesTest method test1.

@Test
public void test1() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.mmap);
    for (final String column : ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().keySet()) {
        final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
        final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
        switch(((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().get(column).getDataType()) {
            case BOOLEAN:
            case STRING:
                Assert.assertTrue(heapDictionary instanceof StringDictionary);
                Assert.assertTrue(mmapDictionary instanceof StringDictionary);
                break;
            case DOUBLE:
                Assert.assertTrue(heapDictionary instanceof DoubleDictionary);
                Assert.assertTrue(mmapDictionary instanceof DoubleDictionary);
                break;
            case FLOAT:
                Assert.assertTrue(heapDictionary instanceof FloatDictionary);
                Assert.assertTrue(mmapDictionary instanceof FloatDictionary);
                break;
            case LONG:
                Assert.assertTrue(heapDictionary instanceof LongDictionary);
                Assert.assertTrue(mmapDictionary instanceof LongDictionary);
                break;
            case INT:
                Assert.assertTrue(heapDictionary instanceof IntDictionary);
                Assert.assertTrue(mmapDictionary instanceof IntDictionary);
                break;
        }
        Assert.assertEquals(mmapDictionary.length(), heapDictionary.length());
        for (int i = 0; i < heapDictionary.length(); i++) {
            Assert.assertEquals(mmapDictionary.get(i), heapDictionary.get(i));
        }
    }
}