Examples with SegmentGeneratorConfig com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig used on opensource projects

Example 31 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testOnlyTableName.

@Test
public void testOnlyTableName() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator(null, "mytable", null, -1);
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable");
}

Example 32 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method CreateSegmentConfigWithoutCreator.

public SegmentGeneratorConfig CreateSegmentConfigWithoutCreator() throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(DefaultSegmentNameGeneratorTest.class.getClassLoader().getResource(AVRO_DATA));
    // Intentionally changed this to TimeUnit.Hours to make it non-default for testing.
    SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), INDEX_DIR, "daysSinceEpoch", TimeUnit.HOURS, "testTable");
    config.setSegmentNamePostfix("1");
    config.setTimeColumnName("daysSinceEpoch");
    return config;
}

Example 33 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testNullPostfixWithNonNegSequenceId.

@Test
public void testNullPostfixWithNonNegSequenceId() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator("daysSinceEpoch", "mytable", null, 2);
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable_1756015683_1756015683_2");
}

Example 34 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class ColumnMetadataTest method createSegmentConfigWithCreator.

public SegmentGeneratorConfig createSegmentConfigWithCreator() throws Exception {
    SegmentGeneratorConfig config = CreateSegmentConfigWithoutCreator();
    config.setCreatorVersion(CREATOR_VERSION);
    return config;
}

Example 35 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class SegmentWithHllIndexCreateHelper method build.

public SegmentIndexCreationDriver build(boolean enableStarTree, HllConfig hllConfig) throws Exception {
    final SegmentGeneratorConfig segmentGenConfig = new SegmentGeneratorConfig(SegmentTestUtils.extractSchemaFromAvroWithoutTime(inputAvro));
    // set other fields in segmentGenConfig
    segmentGenConfig.setInputFilePath(inputAvro.getAbsolutePath());
    segmentGenConfig.setTimeColumnName(timeColumnName);
    segmentGenConfig.setSegmentTimeUnit(timeUnit);
    segmentGenConfig.setFormat(FileFormat.AVRO);
    segmentGenConfig.setSegmentVersion(SegmentVersion.v1);
    segmentGenConfig.setTableName(tableName);
    segmentGenConfig.setOutDir(INDEX_DIR.getAbsolutePath());
    segmentGenConfig.createInvertedIndexForAllColumns();
    segmentGenConfig.setSegmentName(segmentName);
    segmentGenConfig.setSegmentNamePostfix("1");
    if (enableStarTree) {
        setupStarTreeConfig(segmentGenConfig);
        segmentGenConfig.setHllConfig(hllConfig);
    }
    if (hllConfig != null) {
        segmentGenConfig.setHllConfig(hllConfig);
    }
    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(segmentGenConfig);
    /**
     * derived field (hll) is added during the segment build process
     *
     * {@link SegmentIndexCreationDriverImpl#buildStarTree}
     * {@link SegmentIndexCreationDriverImpl#augmentSchemaWithDerivedColumns}
     * {@link SegmentIndexCreationDriverImpl#populateDefaultDerivedColumnValues}
     */
    driver.build();
    LOGGER.info("segmentGenConfig Schema (w/ derived fields): ");
    schema = segmentGenConfig.getSchema();
    printSchema(schema);
    return driver;
}