Examples with SegmentIndexCreationDriverImpl com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl used on opensource projects

Example 6 with SegmentIndexCreationDriverImpl

use of com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl in project pinot by linkedin.

the class DefaultGroupKeyGeneratorTest method setup.

@BeforeClass
private void setup() throws Exception {
    GenericRow[] segmentData = new GenericRow[NUM_ROWS];
    int value = _random.nextInt(MAX_STEP_LENGTH);
    // Generate random values for the segment.
    for (int i = 0; i < UNIQUE_ROWS; i++) {
        Map<String, Object> map = new HashMap<>();
        for (String singleValueColumn : SINGLE_VALUE_COLUMNS) {
            map.put(singleValueColumn, value);
            value += 1 + _random.nextInt(MAX_STEP_LENGTH);
        }
        for (String multiValueColumn : MULTI_VALUE_COLUMNS) {
            int numMultiValues = 1 + _random.nextInt(MAX_NUM_MULTI_VALUES);
            Integer[] values = new Integer[numMultiValues];
            for (int k = 0; k < numMultiValues; k++) {
                values[k] = value;
                value += 1 + _random.nextInt(MAX_STEP_LENGTH);
            }
            map.put(multiValueColumn, values);
        }
        GenericRow genericRow = new GenericRow();
        genericRow.init(map);
        segmentData[i] = genericRow;
    }
    for (int i = UNIQUE_ROWS; i < NUM_ROWS; i += UNIQUE_ROWS) {
        System.arraycopy(segmentData, 0, segmentData, i, UNIQUE_ROWS);
    }
    // Create an index segment with the random values.
    Schema schema = new Schema();
    for (String singleValueColumn : SINGLE_VALUE_COLUMNS) {
        DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(singleValueColumn, FieldSpec.DataType.INT, true);
        schema.addField(dimensionFieldSpec);
    }
    for (String multiValueColumn : MULTI_VALUE_COLUMNS) {
        DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(multiValueColumn, FieldSpec.DataType.INT, false);
        schema.addField(dimensionFieldSpec);
    }
    SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
    FileUtils.deleteQuietly(new File(INDEX_DIR_PATH));
    config.setOutDir(INDEX_DIR_PATH);
    config.setSegmentName(SEGMENT_NAME);
    SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
    driver.init(config, new TestDataRecordReader(schema, segmentData));
    driver.build();
    IndexSegment indexSegment = Loaders.IndexSegment.load(new File(INDEX_DIR_PATH, SEGMENT_NAME), ReadMode.heap);
    // Get a data fetcher for the index segment.
    Map<String, BaseOperator> dataSourceMap = new HashMap<>();
    Map<String, Block> blockMap = new HashMap<>();
    for (String column : indexSegment.getColumnNames()) {
        DataSource dataSource = indexSegment.getDataSource(column);
        dataSourceMap.put(column, dataSource);
        blockMap.put(column, dataSource.getNextBlock());
    }
    // Generate a random test doc id set.
    int num1 = _random.nextInt(50);
    int num2 = num1 + 1 + _random.nextInt(50);
    for (int i = 0; i < 20; i += 2) {
        _testDocIdSet[i] = num1 + 50 * i;
        _testDocIdSet[i + 1] = num2 + 50 * i;
    }
    DataFetcher dataFetcher = new DataFetcher(dataSourceMap);
    DocIdSetBlock docIdSetBlock = new DocIdSetBlock(_testDocIdSet, _testDocIdSet.length);
    ProjectionBlock projectionBlock = new ProjectionBlock(blockMap, new DataBlockCache(dataFetcher), docIdSetBlock);
    _transformBlock = new TransformBlock(projectionBlock, new HashMap<String, BlockValSet>());
}

Example 7 with SegmentIndexCreationDriverImpl

use of com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl in project pinot by linkedin.

the class RealtimeSegmentConverter method build.

public void build(SegmentVersion segmentVersion) throws Exception {
    // lets create a record reader
    RecordReader reader;
    if (sortedColumn == null) {
        reader = new RealtimeSegmentRecordReader(realtimeSegmentImpl, dataSchema);
    } else {
        reader = new RealtimeSegmentRecordReader(realtimeSegmentImpl, dataSchema, sortedColumn);
    }
    SegmentGeneratorConfig genConfig = new SegmentGeneratorConfig(dataSchema);
    if (invertedIndexColumns != null && !invertedIndexColumns.isEmpty()) {
        for (String column : invertedIndexColumns) {
            genConfig.createInvertedIndexForColumn(column);
        }
    }
    if (noDictionaryColumns != null) {
        genConfig.setRawIndexCreationColumns(noDictionaryColumns);
    }
    genConfig.setTimeColumnName(dataSchema.getTimeFieldSpec().getOutgoingTimeColumnName());
    genConfig.setSegmentTimeUnit(dataSchema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType());
    genConfig.setSegmentVersion(segmentVersion);
    genConfig.setTableName(tableName);
    genConfig.setOutDir(outputPath);
    genConfig.setSegmentName(segmentName);
    final SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
    driver.init(genConfig, reader);
    driver.build();
}

Example 8 with SegmentIndexCreationDriverImpl

use of com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl in project pinot by linkedin.

the class RawIndexCreatorTest method buildIndex.

/**
   * Helper method to build a segment containing a single valued string column with RAW (no-dictionary) index.
   *
   * @return Array of string values for the rows in the generated index.
   * @throws Exception
   */
private RecordReader buildIndex(Schema schema) throws Exception {
    SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
    config.setRawIndexCreationColumns(schema.getDimensionNames());
    config.setOutDir(SEGMENT_DIR_NAME);
    config.setSegmentName(SEGMENT_NAME);
    final List<GenericRow> rows = new ArrayList<>();
    for (int row = 0; row < NUM_ROWS; row++) {
        HashMap<String, Object> map = new HashMap<>();
        for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
            Object value;
            FieldSpec.DataType dataType = fieldSpec.getDataType();
            value = getRandomValue(dataType);
            map.put(fieldSpec.getName(), value);
        }
        GenericRow genericRow = new GenericRow();
        genericRow.init(map);
        rows.add(genericRow);
    }
    SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
    RecordReader reader = new TestRecordReader(rows, schema);
    driver.init(config, reader);
    driver.build();
    _segmentDirectory = SegmentDirectory.createFromLocalFS(driver.getOutputDirectory(), ReadMode.mmap);
    _segmentReader = _segmentDirectory.createReader();
    reader.rewind();
    return reader;
}

Example 9 with SegmentIndexCreationDriverImpl

use of com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl in project pinot by linkedin.

the class BaseMultiValueQueriesTest method buildSegment.

@BeforeTest
public void buildSegment() throws Exception {
    FileUtils.deleteQuietly(INDEX_DIR);
    // Get resource file path.
    URL resource = getClass().getClassLoader().getResource(AVRO_DATA);
    Assert.assertNotNull(resource);
    String filePath = resource.getFile();
    // Build the segment schema.
    Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT).addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING).addSingleValueDimension("column5", FieldSpec.DataType.STRING).addMultiValueDimension("column6", FieldSpec.DataType.INT).addMultiValueDimension("column7", FieldSpec.DataType.INT).addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT).addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT).build();
    // Create the segment generator config.
    SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema);
    segmentGeneratorConfig.setInputFilePath(filePath);
    segmentGeneratorConfig.setTableName("testTable");
    segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
    segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9"));
    // Build the index segment.
    SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
    driver.init(segmentGeneratorConfig);
    driver.build();
}

Example 10 with SegmentIndexCreationDriverImpl

use of com.linkedin.pinot.core.segment.creator.impl.SegmentIndexCreationDriverImpl in project pinot by linkedin.

the class BaseSingleValueQueriesTest method buildSegment.

@BeforeTest
public void buildSegment() throws Exception {
    FileUtils.deleteQuietly(INDEX_DIR);
    // Get resource file path.
    URL resource = getClass().getClassLoader().getResource(AVRO_DATA);
    Assert.assertNotNull(resource);
    String filePath = resource.getFile();
    // Build the segment schema.
    Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT).addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING).addSingleValueDimension("column6", FieldSpec.DataType.INT).addSingleValueDimension("column7", FieldSpec.DataType.INT).addSingleValueDimension("column9", FieldSpec.DataType.INT).addSingleValueDimension("column11", FieldSpec.DataType.STRING).addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT).addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT).build();
    // Create the segment generator config.
    SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema);
    segmentGeneratorConfig.setInputFilePath(filePath);
    segmentGeneratorConfig.setTableName("testTable");
    segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath());
    segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column6", "column7", "column11", "column17", "column18"));
    // Build the index segment.
    SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
    driver.init(segmentGeneratorConfig);
    driver.build();
}