Examples with SegmentMetadataImpl com.linkedin.pinot.core.segment.index.SegmentMetadataImpl used on opensource projects

Example 11 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class ColumnIndexDirectoryTestHelper method writeMetadata.

static SegmentMetadataImpl writeMetadata(SegmentVersion version) {
    SegmentMetadataImpl meta = mock(SegmentMetadataImpl.class);
    when(meta.getVersion()).thenReturn(version.toString());
    when(meta.getSegmentVersion()).thenReturn(version);
    when(meta.getDictionaryFileName(anyString(), anyString())).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocationOnMock) throws Throwable {
            return invocationOnMock.getArguments()[0] + ".dict";
        }
    });
    when(meta.getForwardIndexFileName(anyString(), anyString())).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocationOnMock) throws Throwable {
            return invocationOnMock.getArguments()[0] + ".fwd";
        }
    });
    when(meta.getBitmapInvertedIndexFileName(anyString(), anyString())).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocationOnMock) throws Throwable {
            return invocationOnMock.getArguments()[0] + ".ii";
        }
    });
    return meta;
}

Example 12 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class TestStarTreeMetadata method testStarTreeMetadata.

/**
   * Read the StarTree metadata and assert that the actual values in the metadata are as expected.
   *
   * @throws Exception
   */
@Test
public void testStarTreeMetadata() throws Exception {
    String segment = INDEX_DIR_NAME + File.separator + SEGMENT_NAME;
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(new File(segment));
    StarTreeMetadata starTreeMetadata = segmentMetadata.getStarTreeMetadata();
    Assert.assertEquals(starTreeMetadata.getDimensionsSplitOrder(), DIMENSIONS_SPLIT_ORDER);
    Assert.assertEquals(starTreeMetadata.getMaxLeafRecords(), MAX_LEAF_RECORDS);
    Assert.assertEquals(starTreeMetadata.getSkipStarNodeCreationForDimensions(), SKIP_STAR_NODE_CREATION_DIMENSTIONS);
    Assert.assertEquals(starTreeMetadata.getSkipMaterializationCardinality(), SKIP_CARDINALITY_THRESHOLD);
    Assert.assertEquals(starTreeMetadata.getSkipMaterializationForDimensions(), SKIP_MATERIALIZATION_DIMENSIONS);
}

Example 13 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class DictionariesTest method test2.

@Test
public void test2() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.mmap);
    final Map<String, ColumnMetadata> metadataMap = ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
        final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
        final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
        final Set<Object> uniques = uniqueEntries.get(column);
        final List<Object> list = Arrays.asList(uniques.toArray());
        Collections.shuffle(list);
        for (final Object entry : list) {
            Assert.assertEquals(mmapDictionary.indexOf(entry), heapDictionary.indexOf(entry));
            if (!column.equals("pageKey")) {
                Assert.assertFalse(heapDictionary.indexOf(entry) < 0);
                Assert.assertFalse(mmapDictionary.indexOf(entry) < 0);
            }
        }
    }
}

Example 14 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class BenchmarkQueryEngine method startPinot.

@Setup
public void startPinot() throws Exception {
    System.out.println("Using table name " + TABLE_NAME);
    System.out.println("Using data directory " + DATA_DIRECTORY);
    System.out.println("Starting pinot");
    PerfBenchmarkDriverConf conf = new PerfBenchmarkDriverConf();
    conf.setStartBroker(true);
    conf.setStartController(true);
    conf.setStartServer(true);
    conf.setStartZookeeper(true);
    conf.setUploadIndexes(false);
    conf.setRunQueries(false);
    conf.setServerInstanceSegmentTarDir(null);
    conf.setServerInstanceDataDir(DATA_DIRECTORY);
    conf.setConfigureResources(false);
    _perfBenchmarkDriver = new PerfBenchmarkDriver(conf);
    _perfBenchmarkDriver.run();
    Set<String> tables = new HashSet<String>();
    File[] segments = new File(DATA_DIRECTORY, TABLE_NAME).listFiles();
    for (File segmentDir : segments) {
        SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segmentDir);
        if (!tables.contains(segmentMetadata.getTableName())) {
            _perfBenchmarkDriver.configureTable(segmentMetadata.getTableName());
            tables.add(segmentMetadata.getTableName());
        }
        System.out.println("Adding segment " + segmentDir.getAbsolutePath());
        _perfBenchmarkDriver.addSegment(segmentMetadata);
    }
    ZkClient client = new ZkClient("localhost:2191", 10000, 10000, new ZNRecordSerializer());
    ZNRecord record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    while (true) {
        System.out.println("record = " + record);
        Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
        int onlineSegmentCount = 0;
        for (Map<String, String> instancesAndStates : record.getMapFields().values()) {
            for (String state : instancesAndStates.values()) {
                if (state.equals("ONLINE")) {
                    onlineSegmentCount++;
                    break;
                }
            }
        }
        System.out.println(onlineSegmentCount + " segments online out of " + segments.length);
        if (onlineSegmentCount == segments.length) {
            break;
        }
        record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    }
    ranOnce = false;
    System.out.println(_perfBenchmarkDriver.postQuery(QUERY_PATTERNS[queryPattern], optimizationFlags).toString(2));
}

Example 15 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class BitmapPerformanceBenchmark method iterationSpeed.

public static void iterationSpeed(String indexSegmentDir, String column) throws Exception {
    File indexSegment = new File(indexSegmentDir);
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexSegment);
    Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
    Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
    Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
    File bitMapIndexFile = new File(indexSegmentDir, column + ".bitmap.inv");
    ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
    int cardinality = columnMetadata.getCardinality();
    cardinalityMap.put(column, cardinality);
    PinotDataBuffer bitMapDataBuffer = PinotDataBuffer.fromFile(bitMapIndexFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
    BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitMapDataBuffer, cardinality);
    File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
    SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegment, segmentMetadata, ReadMode.mmap);
    SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
    ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
    ImmutableDictionaryReader dictionary = container.getDictionary();
    dictionaryMap.put(column, dictionary);
    // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
    bitMapIndexMap.put(column, bitmapInvertedIndex);
    int dictId = dictionary.indexOf("na.us");
    ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
    Iterator<Integer> iterator = immutable.iterator();
    int count = 0;
    long start = System.currentTimeMillis();
    while (iterator.hasNext()) {
        iterator.next();
        count = count + 1;
    }
    long end = System.currentTimeMillis();
    System.out.println(" matched: " + count + " Time to iterate:" + (end - start));
    bitMapDataBuffer.close();
}