Search in sources :

Example 36 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentPreProcessorTest method testAddColumnMinMaxValue.

@Test
public void testAddColumnMinMaxValue() throws Exception {
    constructSegment();
    IndexLoadingConfigMetadata indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(new PropertiesConfiguration());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.NONE.toString());
    SegmentPreProcessor processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    ColumnMetadata timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    ColumnMetadata dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column1");
    ColumnMetadata metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertNull(timeColumnMetadata.getMinValue());
    Assert.assertNull(timeColumnMetadata.getMaxValue());
    Assert.assertNull(dimensionColumnMetadata.getMinValue());
    Assert.assertNull(dimensionColumnMetadata.getMaxValue());
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata = new IndexLoadingConfigMetadata(new PropertiesConfiguration());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.TIME.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertNull(dimensionColumnMetadata.getMinValue());
    Assert.assertNull(dimensionColumnMetadata.getMaxValue());
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.NON_METRIC.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC");
    Assert.assertNull(metricColumnMetadata.getMinValue());
    Assert.assertNull(metricColumnMetadata.getMaxValue());
    indexLoadingConfigMetadata.setGenerateColumnMinMaxValueMode(ColumnMinMaxValueGeneratorMode.ALL.toString());
    processor = new SegmentPreProcessor(_segmentDirectoryFile, indexLoadingConfigMetadata, null);
    processor.process();
    segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    timeColumnMetadata = segmentMetadata.getColumnMetadataFor("daysSinceEpoch");
    dimensionColumnMetadata = segmentMetadata.getColumnMetadataFor("column5");
    metricColumnMetadata = segmentMetadata.getColumnMetadataFor("count");
    Assert.assertEquals(timeColumnMetadata.getMinValue(), 1756015683);
    Assert.assertEquals(timeColumnMetadata.getMaxValue(), 1756015683);
    Assert.assertEquals(dimensionColumnMetadata.getMinValue(), "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(dimensionColumnMetadata.getMaxValue(), "yQkJTLOQoOqqhkAClgC");
    Assert.assertEquals(metricColumnMetadata.getMinValue(), 890662862);
    Assert.assertEquals(metricColumnMetadata.getMaxValue(), 890662862);
}
Also used : ColumnMetadata(com.linkedin.pinot.core.segment.index.ColumnMetadata) SegmentMetadataImpl(com.linkedin.pinot.core.segment.index.SegmentMetadataImpl) IndexLoadingConfigMetadata(com.linkedin.pinot.common.metadata.segment.IndexLoadingConfigMetadata) PropertiesConfiguration(org.apache.commons.configuration.PropertiesConfiguration) Test(org.testng.annotations.Test)

Aggregations

SegmentMetadataImpl (com.linkedin.pinot.core.segment.index.SegmentMetadataImpl)36 File (java.io.File)18 Test (org.testng.annotations.Test)13 ColumnMetadata (com.linkedin.pinot.core.segment.index.ColumnMetadata)10 SegmentMetadata (com.linkedin.pinot.common.segment.SegmentMetadata)8 SegmentDirectory (com.linkedin.pinot.core.segment.store.SegmentDirectory)8 IndexSegment (com.linkedin.pinot.core.indexsegment.IndexSegment)5 PinotDataBuffer (com.linkedin.pinot.core.segment.memory.PinotDataBuffer)5 OfflineSegmentZKMetadata (com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata)4 FileTime (java.nio.file.attribute.FileTime)4 IndexSegmentImpl (com.linkedin.pinot.core.segment.index.IndexSegmentImpl)3 SegmentV1V2ToV3FormatConverter (com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter)3 ImmutableDictionaryReader (com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader)3 ArrayList (java.util.ArrayList)3 AbstractTableConfig (com.linkedin.pinot.common.config.AbstractTableConfig)2 SingleColumnMultiValueReader (com.linkedin.pinot.core.io.reader.SingleColumnMultiValueReader)2 SingleColumnSingleValueReader (com.linkedin.pinot.core.io.reader.SingleColumnSingleValueReader)2 ColumnIndexContainer (com.linkedin.pinot.core.segment.index.column.ColumnIndexContainer)2 BitmapInvertedIndexReader (com.linkedin.pinot.core.segment.index.readers.BitmapInvertedIndexReader)2 StringDictionary (com.linkedin.pinot.core.segment.index.readers.StringDictionary)2