Examples with SegmentIndexCreationDriver com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver used on opensource projects

Example 21 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method setUP.

@BeforeClass
public void setUP() throws Exception {
    if (INDEX_DIR.exists()) {
        FileUtils.deleteQuietly(INDEX_DIR);
    }
    final String filePath = TestUtils.getFileFromResourceUrl(ChunkIndexCreationDriverImplTest.class.getClassLoader().getResource(AVRO_DATA));
    final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), INDEX_DIR, "daysSinceEpoch", TimeUnit.DAYS, "testTable");
    config.setSegmentNamePostfix("1");
    config.setTimeColumnName("daysSinceEpoch");
    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
}

Example 22 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class QueriesSentinelTest method setupSegmentFor.

private void setupSegmentFor(String table) throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
    if (INDEX_DIR.exists()) {
        FileUtils.deleteQuietly(INDEX_DIR);
    }
    INDEX_DIR.mkdir();
    final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), new File(INDEX_DIR, "segment"), "daysSinceEpoch", TimeUnit.DAYS, table);
    final SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
    driver.init(config);
    driver.build();
//    System.out.println("built at : " + INDEX_DIR.getAbsolutePath());
}

Example 23 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class HelixStarterTest method setupSegment.

private void setupSegment(File segmentDir, String tableName) throws Exception {
    String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
    SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), segmentDir, TimeUnit.DAYS, tableName, null);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    LOGGER.info("Table: {} built at path: {}", tableName, segmentDir.getAbsolutePath());
}

Example 24 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class IntegrationTest method setupSegmentList.

private void setupSegmentList() throws Exception {
    final URL resource = getClass().getClassLoader().getResource(SMALL_AVRO_DATA);
    final String filePath = TestUtils.getFileFromResourceUrl(resource);
    _indexSegmentList.clear();
    if (INDEXES_DIR.exists()) {
        FileUtils.deleteQuietly(INDEXES_DIR);
    }
    INDEXES_DIR.mkdir();
    for (int i = 0; i < 2; ++i) {
        final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
        final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "testTable");
        final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
        driver.init(config);
        driver.build();
        _indexSegmentList.add(ColumnarSegmentLoader.load(new File(new File(INDEXES_DIR, "segment_" + String.valueOf(i)), driver.getSegmentName()), ReadMode.mmap));
    //      System.out.println("built at : " + segmentDir.getAbsolutePath());
    }
}

Example 25 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class BaseMultiValueQueriesTest method buildSegment.

@BeforeTest
public void buildSegment() throws Exception {
    FileUtils.deleteQuietly(INDEX_DIR);
    // Get resource file path.
    URL resource = getClass().getClassLoader().getResource(AVRO_DATA);
    Assert.assertNotNull(resource);
    String filePath = resource.getFile();
    // Build the segment schema.
    Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT).addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING).addSingleValueDimension("column5", FieldSpec.DataType.STRING).addMultiValueDimension("column6", FieldSpec.DataType.INT).addMultiValueDimension("column7", FieldSpec.DataType.INT).addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT).addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT).build();
    // Create the segment generator config.
    SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema);
    segmentGeneratorConfig.setInputFilePath(filePath);
    segmentGeneratorConfig.setTableName("testTable");
    segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
    segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9"));
    // Build the index segment.
    SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
    driver.init(segmentGeneratorConfig);
    driver.build();
}