Examples with SegmentIndexCreationDriver com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver used on opensource projects

Example 11 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class ColumnMetadataTest method testPaddingCharacter.

@Test
public void testPaddingCharacter() throws Exception {
    // Build the Segment metadata.
    SegmentGeneratorConfig config = CreateSegmentConfigWithoutCreator();
    config.setPaddingCharacter('\0');
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    // Load segment metadata.
    IndexSegment segment = Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    SegmentMetadataImpl metadata = (SegmentMetadataImpl) segment.getSegmentMetadata();
    verifySegmentAfterLoading(metadata);
    // Make sure we get null for creator name.
    char paddingCharacter = metadata.getPaddingCharacter();
    Assert.assertEquals(paddingCharacter, '\0');
}

Example 12 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class QueryExceptionTest method setupSegmentFor.

private void setupSegmentFor(String table) throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
    if (INDEX_DIR.exists()) {
        FileUtils.deleteQuietly(INDEX_DIR);
    }
    INDEX_DIR.mkdir();
    final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), new File(INDEX_DIR, "segment"), "daysSinceEpoch", TimeUnit.DAYS, table);
    final SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
    driver.init(config);
    driver.build();
//    System.out.println("built at : " + INDEX_DIR.getAbsolutePath());
}

Example 13 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class QueryExecutorTest method setupSegmentList.

private void setupSegmentList(int numberOfSegments) throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
    _indexSegmentList.clear();
    if (INDEXES_DIR.exists()) {
        FileUtils.deleteQuietly(INDEXES_DIR);
    }
    INDEXES_DIR.mkdir();
    for (int i = 0; i < numberOfSegments; ++i) {
        final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
        final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas");
        config.setSegmentNamePostfix(String.valueOf(i));
        final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
        driver.init(config);
        driver.build();
        File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i));
        String segmentName = parent.list()[0];
        _indexSegmentList.add(ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap));
    //      System.out.println("built at : " + segmentDir.getAbsolutePath());
    }
}

Example 14 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class BrokerReduceServiceTest method setupSegmentList.

private void setupSegmentList(int numberOfSegments) throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
    _indexSegmentList.clear();
    if (INDEXES_DIR.exists()) {
        FileUtils.deleteQuietly(INDEXES_DIR);
    }
    INDEXES_DIR.mkdir();
    for (int i = 0; i < numberOfSegments; ++i) {
        final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
        final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas");
        config.setSegmentNamePostfix(String.valueOf(i));
        final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
        driver.init(config);
        driver.build();
        File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i));
        String segmentName = parent.list()[0];
        _indexSegmentList.add(ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap));
    //      System.out.println("built at : " + segmentDir.getAbsolutePath());
    }
}

Example 15 with SegmentIndexCreationDriver

use of com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver in project pinot by linkedin.

the class BitmapInvertedIndexTest method setup.

@BeforeClass
public void setup() throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
    if (INDEX_DIR.exists()) {
        FileUtils.deleteQuietly(INDEX_DIR);
    }
    final SegmentGeneratorConfig config = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test");
    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    List<String> iiColumns = config.getInvertedIndexCreationColumns();
    invertedIndexColumns = new String[iiColumns.size()];
    iiColumns.toArray(invertedIndexColumns);
    segmentDirectory = new File(INDEX_DIR, driver.getSegmentName());
//    System.out.println("built at : " + INDEX_DIR.getAbsolutePath());
}