Examples with SegmentGeneratorConfig com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig used on opensource projects

Example 1 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class AvroDataPublisherTest method TestReadAvro.

@Test
public void TestReadAvro() throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
    final String jsonPath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(JSON_DATA));
    Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("column3", DataType.STRING).addSingleValueDimension("column2", DataType.STRING).build();
    final SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
    config.setFormat(FileFormat.AVRO);
    config.setInputFilePath(filePath);
    config.setSegmentVersion(SegmentVersion.v1);
    AvroRecordReader avroDataPublisher = (AvroRecordReader) RecordReaderFactory.get(config);
    int cnt = 0;
    for (String line : FileUtils.readLines(new File(jsonPath))) {
        JSONObject obj = new JSONObject(line);
        if (avroDataPublisher.hasNext()) {
            GenericRow recordRow = avroDataPublisher.next();
            for (String column : recordRow.getFieldNames()) {
                String valueFromJson = obj.get(column).toString();
                String valueFromAvro = recordRow.getValue(column).toString();
                if (cnt > 1) {
                    Assert.assertEquals(valueFromJson, valueFromAvro);
                }
            }
        }
        cnt++;
    }
    Assert.assertEquals(cnt, 10001);
}

Example 2 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class AvroDataPublisherTest method TestReadMultiValueAvro.

@Test
public void TestReadMultiValueAvro() throws Exception {
    final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_MULTI_DATA));
    final SegmentGeneratorConfig config = new SegmentGeneratorConfig(AvroUtils.extractSchemaFromAvro(new File(filePath)));
    config.setFormat(FileFormat.AVRO);
    config.setInputFilePath(filePath);
    config.setSegmentVersion(SegmentVersion.v1);
    AvroRecordReader avroDataPublisher = (AvroRecordReader) RecordReaderFactory.get(config);
    int cnt = 0;
    while (avroDataPublisher.hasNext()) {
        GenericRow recordRow = avroDataPublisher.next();
        for (String column : recordRow.getFieldNames()) {
            String valueStringFromAvro = null;
            if (avroDataPublisher.getSchema().getFieldSpecFor(column).isSingleValueField()) {
                Object valueFromAvro = recordRow.getValue(column);
                valueStringFromAvro = valueFromAvro.toString();
            } else {
                Object[] valueFromAvro = (Object[]) recordRow.getValue(column);
                valueStringFromAvro = "[";
                int i = 0;
                for (Object valueObject : valueFromAvro) {
                    if (i++ == 0) {
                        valueStringFromAvro += valueObject.toString();
                    } else {
                        valueStringFromAvro += ", " + valueObject.toString();
                    }
                }
                valueStringFromAvro += "]";
            }
        }
        cnt++;
    }
    Assert.assertEquals(28949, cnt);
}

Example 3 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testNullPostfix.

@Test
public void testNullPostfix() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator("daysSinceEpoch", "mytable", null, -1);
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable_1756015683_1756015683");
}

Example 4 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testPostfix.

@Test
public void testPostfix() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator("daysSinceEpoch", "mytable", "1", -1);
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable_1756015683_1756015683_1");
}

Example 5 with SegmentGeneratorConfig

use of com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig in project pinot by linkedin.

the class DefaultSegmentNameGeneratorTest method testAlreadyNamedSegment.

@Test
public void testAlreadyNamedSegment() throws Exception {
    ColumnMetadataTest columnMetadataTest = new ColumnMetadataTest();
    // Build the Segment metadata.
    SegmentGeneratorConfig config = columnMetadataTest.CreateSegmentConfigWithoutCreator();
    SegmentNameGenerator segmentNameGenerator = new DefaultSegmentNameGenerator("mytable_1");
    config.setSegmentNameGenerator(segmentNameGenerator);
    SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();
    Assert.assertEquals(driver.getSegmentName(), "mytable_1");
}