Examples with IndexSegmentImpl com.linkedin.pinot.core.segment.index.IndexSegmentImpl used on opensource projects

Example 6 with IndexSegmentImpl

use of com.linkedin.pinot.core.segment.index.IndexSegmentImpl in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test3.

@Test
public void test3() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final DataSource ds = segment.getDataSource("column7");
    final Block bl = ds.nextBlock();
    final BlockValSet valSet = bl.getBlockValueSet();
    final int maxValue = ((SegmentMetadataImpl) segment.getSegmentMetadata()).getColumnMetadataFor("column7").getMaxNumberOfMultiValues();
    final BlockMultiValIterator it = (BlockMultiValIterator) valSet.iterator();
    while (it.hasNext()) {
        final int[] entry = new int[maxValue];
        it.nextIntVal(entry);
        LOGGER.trace(Arrays.toString(entry));
    }
}

Example 7 with IndexSegmentImpl

use of com.linkedin.pinot.core.segment.index.IndexSegmentImpl in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test5.

@Test(enabled = false)
public void test5() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final List<String> rhs = new ArrayList<String>();
    rhs.add("-100");
    final Predicate p = new EqPredicate("column1", rhs);
    final DataSource ds = segment.getDataSource("column1", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}

Example 8 with IndexSegmentImpl

use of com.linkedin.pinot.core.segment.index.IndexSegmentImpl in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test2.

@Test
public void test2() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    //    System.out.println("INdex dir:" + INDEX_DIR);
    final DataSource ds = segment.getDataSource("column1");
    final Block bl = ds.nextBlock();
    final BlockValSet valSet = bl.getBlockValueSet();
    final BlockSingleValIterator it = (BlockSingleValIterator) valSet.iterator();
    // TODO: FIXME - load segment with known data and verify that it exists
    while (it.hasNext()) {
        LOGGER.trace(Integer.toString(it.nextIntVal()));
    }
}

Example 9 with IndexSegmentImpl

use of com.linkedin.pinot.core.segment.index.IndexSegmentImpl in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test6.

@Test(enabled = false)
public void test6() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final ImmutableDictionaryReader d = segment.getDictionaryFor("column7");
    final List<String> rhs = new ArrayList<String>();
    rhs.add(d.get(new Random().nextInt(d.length())).toString());
    final Predicate p = new EqPredicate("column7", rhs);
    final DataSource ds = segment.getDataSource("column7", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}

Example 10 with IndexSegmentImpl

use of com.linkedin.pinot.core.segment.index.IndexSegmentImpl in project pinot by linkedin.

the class FilterOperatorBenchmark method main.

public static void main(String[] args) throws Exception {
    String rootDir = args[0];
    File[] segmentDirs = new File(rootDir).listFiles();
    String query = args[1];
    AtomicInteger totalDocsMatched = new AtomicInteger(0);
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = pql2Compiler.compileToBrokerRequest(query);
    List<Callable<Void>> segmentProcessors = new ArrayList<>();
    long[] timesSpent = new long[segmentDirs.length];
    for (int i = 0; i < segmentDirs.length; i++) {
        File indexSegmentDir = segmentDirs[i];
        System.out.println("Loading " + indexSegmentDir.getName());
        Configuration tableDataManagerConfig = new PropertiesConfiguration();
        List<String> invertedColumns = new ArrayList<>();
        FilenameFilter filter = new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return name.endsWith(".bitmap.inv");
            }
        };
        String[] indexFiles = indexSegmentDir.list(filter);
        for (String indexFileName : indexFiles) {
            invertedColumns.add(indexFileName.replace(".bitmap.inv", ""));
        }
        tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, invertedColumns);
        IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(tableDataManagerConfig);
        IndexSegmentImpl indexSegmentImpl = (IndexSegmentImpl) Loaders.IndexSegment.load(indexSegmentDir, ReadMode.heap, indexLoadingConfigMetadata);
        segmentProcessors.add(new SegmentProcessor(i, indexSegmentImpl, brokerRequest, totalDocsMatched, timesSpent));
    }
    ExecutorService executorService = Executors.newCachedThreadPool();
    for (int run = 0; run < 5; run++) {
        System.out.println("START RUN:" + run);
        totalDocsMatched.set(0);
        long start = System.currentTimeMillis();
        List<Future<Void>> futures = executorService.invokeAll(segmentProcessors);
        for (int i = 0; i < futures.size(); i++) {
            futures.get(i).get();
        }
        long end = System.currentTimeMillis();
        System.out.println("Total docs matched:" + totalDocsMatched + " took:" + (end - start));
        System.out.println("Times spent:" + Arrays.toString(timesSpent));
        System.out.println("END RUN:" + run);
    }
    System.exit(0);
}