Examples with StarTreeInterf com.linkedin.pinot.core.startree.StarTreeInterf used on opensource projects

Example 1 with StarTreeInterf

use of com.linkedin.pinot.core.startree.StarTreeInterf in project pinot by linkedin.

the class SegmentDumpTool method doMain.

public void doMain(String[] args) throws Exception {
    CmdLineParser parser = new CmdLineParser(this);
    parser.parseArgument(args);
    File segmentDir = new File(segmentPath);
    SegmentMetadata metadata = new SegmentMetadataImpl(segmentDir);
    // All columns by default
    if (columnNames == null) {
        columnNames = new ArrayList<String>(metadata.getSchema().getColumnNames());
        Collections.sort(columnNames);
    }
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDir, ReadMode.mmap);
    Map<String, Dictionary> dictionaries = new HashMap<String, Dictionary>();
    Map<String, BlockSingleValIterator> iterators = new HashMap<String, BlockSingleValIterator>();
    for (String columnName : columnNames) {
        DataSource dataSource = indexSegment.getDataSource(columnName);
        dataSource.open();
        Block block = dataSource.nextBlock();
        BlockValSet blockValSet = block.getBlockValueSet();
        BlockSingleValIterator itr = (BlockSingleValIterator) blockValSet.iterator();
        iterators.put(columnName, itr);
        dictionaries.put(columnName, dataSource.getDictionary());
    }
    System.out.print("Doc\t");
    for (String columnName : columnNames) {
        System.out.print(columnName);
        System.out.print("\t");
    }
    System.out.println();
    for (int i = 0; i < indexSegment.getSegmentMetadata().getTotalDocs(); i++) {
        System.out.print(i);
        System.out.print("\t");
        for (String columnName : columnNames) {
            FieldSpec.DataType columnType = metadata.getSchema().getFieldSpecFor(columnName).getDataType();
            BlockSingleValIterator itr = iterators.get(columnName);
            Integer encodedValue = itr.nextIntVal();
            Object value = dictionaries.get(columnName).get(encodedValue);
            System.out.print(value);
            System.out.print("\t");
        }
        System.out.println();
    }
    if (dumpStarTree) {
        System.out.println();
        File starTreeFile = new File(segmentDir, V1Constants.STAR_TREE_INDEX_FILE);
        StarTreeInterf tree = StarTreeSerDe.fromFile(starTreeFile, ReadMode.mmap);
        tree.printTree();
    }
}

Example 2 with StarTreeInterf

use of com.linkedin.pinot.core.startree.StarTreeInterf in project pinot by linkedin.

the class StarTreeOnHeapToOffHeapConverter method execute.

@Override
public boolean execute() throws Exception {
    File indexDir = new File(_segmentDir);
    long start = System.currentTimeMillis();
    LOGGER.info("Loading segment {}", indexDir.getName());
    IndexSegment segment = Loaders.IndexSegment.load(indexDir, ReadMode.heap);
    long end = System.currentTimeMillis();
    LOGGER.info("Loaded segment {} in {} ms ", indexDir.getName(), (end - start));
    start = end;
    StarTreeInterf starTreeOnHeap = segment.getStarTree();
    File starTreeOffHeapFile = new File(TMP_DIR, (V1Constants.STAR_TREE_INDEX_FILE + System.currentTimeMillis()));
    // Convert the star tree on-heap to off-heap format.
    StarTreeSerDe.writeTreeOffHeapFormat(starTreeOnHeap, starTreeOffHeapFile);
    // Copy all the indexes into output directory.
    File outputDir = new File(_outputDir);
    FileUtils.deleteQuietly(outputDir);
    FileUtils.copyDirectory(indexDir, outputDir);
    // Delete the existing star tree on-heap file from the output directory.
    FileUtils.deleteQuietly(new File(_outputDir, V1Constants.STAR_TREE_INDEX_FILE));
    // Move the temp star tree off-heap file into the output directory.
    FileUtils.moveFile(starTreeOffHeapFile, new File(_outputDir, V1Constants.STAR_TREE_INDEX_FILE));
    end = System.currentTimeMillis();
    LOGGER.info("Converted segment: {} ms", (end - start));
    return true;
}