Examples with SegmentMetadataImpl com.linkedin.pinot.core.segment.index.SegmentMetadataImpl used on opensource projects

Example 26 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class DictionariesTest method test1.

@Test
public void test1() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(segmentDirectory, ReadMode.mmap);
    for (final String column : ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().keySet()) {
        final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
        final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
        switch(((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().get(column).getDataType()) {
            case BOOLEAN:
            case STRING:
                Assert.assertTrue(heapDictionary instanceof StringDictionary);
                Assert.assertTrue(mmapDictionary instanceof StringDictionary);
                break;
            case DOUBLE:
                Assert.assertTrue(heapDictionary instanceof DoubleDictionary);
                Assert.assertTrue(mmapDictionary instanceof DoubleDictionary);
                break;
            case FLOAT:
                Assert.assertTrue(heapDictionary instanceof FloatDictionary);
                Assert.assertTrue(mmapDictionary instanceof FloatDictionary);
                break;
            case LONG:
                Assert.assertTrue(heapDictionary instanceof LongDictionary);
                Assert.assertTrue(mmapDictionary instanceof LongDictionary);
                break;
            case INT:
                Assert.assertTrue(heapDictionary instanceof IntDictionary);
                Assert.assertTrue(mmapDictionary instanceof IntDictionary);
                break;
        }
        Assert.assertEquals(mmapDictionary.length(), heapDictionary.length());
        for (int i = 0; i < heapDictionary.length(); i++) {
            Assert.assertEquals(mmapDictionary.get(i), heapDictionary.get(i));
        }
    }
}

Example 27 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class IntArraysTest method test1.

@Test
public void test1() throws Exception {
    final IndexSegmentImpl heapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.heap);
    final IndexSegmentImpl mmapSegment = (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final Map<String, ColumnMetadata> metadataMap = ((SegmentMetadataImpl) heapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
        final DataFileReader heapArray = heapSegment.getForwardIndexReaderFor(column);
        final DataFileReader mmapArray = mmapSegment.getForwardIndexReaderFor(column);
        if (metadataMap.get(column).isSingleValue()) {
            final SingleColumnSingleValueReader svHeapReader = (SingleColumnSingleValueReader) heapArray;
            final SingleColumnSingleValueReader mvMmapReader = (SingleColumnSingleValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                Assert.assertEquals(mvMmapReader.getInt(i), svHeapReader.getInt(i));
            }
        } else {
            final SingleColumnMultiValueReader svHeapReader = (SingleColumnMultiValueReader) heapArray;
            final SingleColumnMultiValueReader mvMmapReader = (SingleColumnMultiValueReader) mmapArray;
            for (int i = 0; i < metadataMap.get(column).getTotalDocs(); i++) {
                final int[] i_1 = new int[1000];
                final int[] j_i = new int[1000];
                Assert.assertEquals(mvMmapReader.getIntArray(i, j_i), svHeapReader.getIntArray(i, i_1));
            }
        }
    }
}

Example 28 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class ValidationManagerTest method testPushTimePersistence.

@Test
public void testPushTimePersistence() throws Exception {
    DummyMetadata metadata = new DummyMetadata(TEST_TABLE_NAME);
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    SegmentMetadata fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    long pushTime = fetchedMetadata.getPushTime();
    // Check that the segment has been pushed in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - pushTime < 30000);
    // Check that there is no refresh time
    Assert.assertEquals(fetchedMetadata.getRefreshTime(), Long.MIN_VALUE);
    // Refresh the segment
    metadata.setCrc("anotherfakecrc");
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    // Check that the segment still has the same push time
    Assert.assertEquals(fetchedMetadata.getPushTime(), pushTime);
    // Check that the refresh time is in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - fetchedMetadata.getRefreshTime() < 30000);
}

Example 29 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverterTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentMetadataImpl beforeConversionMeta = new SegmentMetadataImpl(segmentDirectory);
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(segmentDirectory);
    File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
    Assert.assertTrue(v3Location.exists());
    Assert.assertTrue(v3Location.isDirectory());
    Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
    SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
    Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
    Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
    // Drop the star tree index file because it has invalid data
    new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
    new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
    FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
    // verify that the segment loads correctly. This is necessary and sufficient
    // full proof way to ensure that segment is correctly translated
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
    Assert.assertNotNull(indexSegment);
    Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
    Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    FileTime afterLoadTime = Files.getLastModifiedTime(v3Location.toPath());
    Assert.assertEquals(afterConversionTime, afterLoadTime);
    // verify that SegmentMetadataImpl loaded from segmentDirectory correctly sets
    // metadata information after conversion. This has impacted us while loading
    // segments by triggering download. That's costly. That's also difficult to test
    Assert.assertFalse(new File(segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME).exists());
    SegmentMetadataImpl metaAfterConversion = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertNotNull(metaAfterConversion);
    Assert.assertFalse(metaAfterConversion.getCrc().equalsIgnoreCase(String.valueOf(Long.MIN_VALUE)));
    Assert.assertEquals(metaAfterConversion.getCrc(), beforeConversionMeta.getCrc());
    Assert.assertTrue(metaAfterConversion.getIndexCreationTime() != Long.MIN_VALUE);
    Assert.assertEquals(metaAfterConversion.getIndexCreationTime(), beforeConversionMeta.getIndexCreationTime());
}

Example 30 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class LoadersTest method testLoad.

@Test
public void testLoad() throws Exception {
    SegmentMetadataImpl originalMetadata = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertEquals(originalMetadata.getSegmentVersion(), SegmentVersion.v1);
    // note: ordering of these two test blocks matters
    {
        // Explicitly pass v1 format since we will convert by default to v3
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v1LoadingConfig);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), originalMetadata.getVersion());
        Assert.assertFalse(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
    {
        // with this code and converter in place, make sure we still load original version
        // by default. We require specific configuration for v3.
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
}