Examples with SegmentIndexCreationDriver com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver used on opensource projects

Example 1 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class ResourceTestHelper method setupSegment.

public IndexSegment setupSegment(String tableName, String avroDataFilePath, String segmentNamePostfix) throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(SegmentV1V2ToV3FormatConverter.class.getClassLoader().getResource(avroDataFilePath));
    // intentionally changed this to TimeUnit.Hours to make it non-default for testing
    final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), INDEX_DIR, "daysSinceEpoch", TimeUnit.HOURS, tableName);
    config.setSegmentNamePostfix(segmentNamePostfix);
    config.setTimeColumnName("daysSinceEpoch");
    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    File segmentDirectory = new File(INDEX_DIR, driver.getSegmentName());
    IndexSegment segment = ColumnarSegmentLoader.load(segmentDirectory, ReadMode.mmap);
    serverInstance.getInstanceDataManager().addSegment(segment.getSegmentMetadata(), null, null);
    return segment;
}

Example 2 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class ColumnarToStarTreeConverter method convertSegment.

/**
   * Helper method to perform the conversion.
   * @param columnarSegment Columnar segment directory to convert
   * @throws Exception
   */
private void convertSegment(File columnarSegment) throws Exception {
    PinotSegmentRecordReader pinotSegmentRecordReader = new PinotSegmentRecordReader(columnarSegment);
    SegmentGeneratorConfig config = new SegmentGeneratorConfig(pinotSegmentRecordReader.getSchema());
    config.setDataDir(_inputDirName);
    config.setInputFilePath(columnarSegment.getAbsolutePath());
    config.setFormat(FileFormat.PINOT);
    config.setEnableStarTreeIndex(true);
    config.setOutDir(_outputDirName);
    config.setStarTreeIndexSpecFile(_starTreeConfigFileName);
    config.setOverwrite(_overwrite);
    config.setSegmentName(columnarSegment.getName());
    SegmentIndexCreationDriver indexCreator = new SegmentIndexCreationDriverImpl();
    indexCreator.init(config);
    indexCreator.build();
}

Example 3 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class HllIndexCreationTest method testColumnStatsWithStarTree.

@Test
public void testColumnStatsWithStarTree() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    boolean hasException = false;
    int maxDocLength = 10000;
    try {
        LOGGER.debug("================ With StarTree ================");
        helper = new SegmentWithHllIndexCreateHelper("withStarTree", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        LOGGER.debug("================ Cardinality ================");
        for (String name : helper.getSchema().getColumnNames()) {
            LOGGER.debug("* " + name + ": " + driver.getColumnStatisticsCollector(name).getCardinality());
        }
        LOGGER.debug("Loading ...");
        IndexSegment indexSegment = Loaders.IndexSegment.load(helper.getSegmentDirectory(), ReadMode.mmap);
        int[] docIdSet = new int[maxDocLength];
        for (int i = 0; i < maxDocLength; i++) {
            docIdSet[i] = i;
        }
        Map<String, BaseOperator> dataSourceMap = new HashMap<>();
        for (String column : indexSegment.getColumnNames()) {
            dataSourceMap.put(column, indexSegment.getDataSource(column));
        }
        DataBlockCache blockCache = new DataBlockCache(new DataFetcher(dataSourceMap));
        blockCache.initNewBlock(docIdSet, 0, maxDocLength);
        String[] strings = blockCache.getStringValueArrayForColumn("column1_hll");
        Assert.assertEquals(strings.length, maxDocLength);
        double[] ints = blockCache.getDoubleValueArrayForColumn("column1");
        Assert.assertEquals(ints.length, maxDocLength);
    } catch (Exception e) {
        hasException = true;
        LOGGER.error(e.getMessage());
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
        Assert.assertEquals(hasException, false);
    }
}

Example 4 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class HllIndexCreationTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    try {
        helper = new SegmentWithHllIndexCreateHelper("testConvert", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        File segmentDirectory = new File(helper.getIndexDir(), driver.getSegmentName());
        LOGGER.debug("Segment Directory: " + segmentDirectory.getAbsolutePath());
        SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
        converter.convert(segmentDirectory);
        File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
        LOGGER.debug("v3Location: " + v3Location.getAbsolutePath());
        Assert.assertTrue(v3Location.exists());
        Assert.assertTrue(v3Location.isDirectory());
        Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
        SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
        LOGGER.debug("metadata all columns: " + metadata.getAllColumns());
        Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
        // Drop the star tree index file because it has invalid data
        // new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
        // new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
        FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
        // verify that the segment loads correctly. This is necessary and sufficient
        // full proof way to ensure that segment is correctly translated
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
        Assert.assertNotNull(indexSegment);
        Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
        Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
    }
}

Example 5 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testNullPostfix.

@Test
public void testNullPostfix() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator("daysSinceEpoch", "mytable", null, -1);
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable_1756015683_1756015683");
}