Examples with SegmentIndexCreationInfo com.linkedin.pinot.core.segment.creator.SegmentIndexCreationInfo used on opensource projects

Example 1 with SegmentIndexCreationInfo

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationInfo in project pinot by linkedin.

the class SegmentIndexCreationDriverImpl method init.

public void init(SegmentGeneratorConfig config, SegmentCreationDataSource dataSource) throws Exception {
    this.config = config;
    this.createStarTree = config.isEnableStarTreeIndex();
    recordReader = dataSource.getRecordReader();
    dataSchema = recordReader.getSchema();
    if (config.getHllConfig() != null) {
        HllConfig hllConfig = config.getHllConfig();
        // generate HLL fields
        if (hllConfig.getColumnsToDeriveHllFields() != null && !hllConfig.getColumnsToDeriveHllFields().isEmpty()) {
            if (!createStarTree) {
                throw new IllegalArgumentException("Derived HLL fields generation will not work if StarTree is not enabled.");
            } else {
                createHllIndex = true;
            }
        }
    // else columnsToDeriveHllFields is null...don't do anything in this case
    // segment seal() will write the log2m value to the metadata
    }
    addDerivedFieldsInSchema();
    extractor = FieldExtractorFactory.getPlainFieldExtractor(dataSchema);
    // Initialize stats collection
    if (!createStarTree) {
        // For star tree, the stats are gathered in buildStarTree()
        segmentStats = dataSource.gatherStats(extractor);
        totalDocs = segmentStats.getTotalDocCount();
        totalRawDocs = segmentStats.getRawDocCount();
        totalAggDocs = segmentStats.getAggregatedDocCount();
    }
    // Initialize index creation
    segmentIndexCreationInfo = new SegmentIndexCreationInfo();
    indexCreationInfoMap = new HashMap<>();
    // Check if has star tree
    indexCreator = new SegmentColumnarIndexCreator();
    // Ensure that the output directory exists
    final File indexDir = new File(config.getOutDir());
    if (!indexDir.exists()) {
        indexDir.mkdirs();
    }
    // Create a temporary directory used in segment creation
    tempIndexDir = new File(indexDir, com.linkedin.pinot.common.utils.FileUtils.getRandomFileName());
    starTreeTempDir = new File(indexDir, com.linkedin.pinot.common.utils.FileUtils.getRandomFileName());
    LOGGER.debug("tempIndexDir:{}", tempIndexDir);
    LOGGER.debug("starTreeTempDir:{}", starTreeTempDir);
}