Examples with SegmentV1V2ToV3FormatConverter com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter used on opensource projects

Example 1 with SegmentV1V2ToV3FormatConverter

use of com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter in project pinot by linkedin.

the class HllIndexCreationTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    try {
        helper = new SegmentWithHllIndexCreateHelper("testConvert", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        File segmentDirectory = new File(helper.getIndexDir(), driver.getSegmentName());
        LOGGER.debug("Segment Directory: " + segmentDirectory.getAbsolutePath());
        SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
        converter.convert(segmentDirectory);
        File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
        LOGGER.debug("v3Location: " + v3Location.getAbsolutePath());
        Assert.assertTrue(v3Location.exists());
        Assert.assertTrue(v3Location.isDirectory());
        Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
        SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
        LOGGER.debug("metadata all columns: " + metadata.getAllColumns());
        Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
        // Drop the star tree index file because it has invalid data
        // new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
        // new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
        FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
        // verify that the segment loads correctly. This is necessary and sufficient
        // full proof way to ensure that segment is correctly translated
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
        Assert.assertNotNull(indexSegment);
        Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
        Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
    }
}

Example 2 with SegmentV1V2ToV3FormatConverter

use of com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter in project pinot by linkedin.

the class SegmentPreProcessorTest method testV3UpdateDefaultColumns.

@Test
public void testV3UpdateDefaultColumns() throws Exception {
    constructSegment();
    // Convert segment format to v3.
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(_segmentDirectoryFile);
    File v3SegmentDirectoryFile = new File(_segmentDirectoryFile, V3_SEGMENT_NAME);
    checkUpdateDefaultColumns(v3SegmentDirectoryFile);
    // is not the same length as before. This should throw exception for segment format v3.
    try {
        SegmentPreProcessor processor = new SegmentPreProcessor(v3SegmentDirectoryFile, _indexLoadingConfigMetadata, _newColumnsSchema3);
        processor.process();
        Assert.fail("For segment format v3, should throw exception when trying to update with different length index");
    } catch (Exception e) {
    // PASS.
    }
}

Example 3 with SegmentV1V2ToV3FormatConverter

use of com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter in project pinot by linkedin.

the class LoadersTest method testLoadWithStaleConversionDir.

@Test
public void testLoadWithStaleConversionDir() throws Exception {
    // Format converter will leave stale directories around if there was
    // conversion failure. This test case checks loading in that scenario
    SegmentMetadataImpl originalMetadata = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertEquals(originalMetadata.getSegmentVersion(), SegmentVersion.v1);
    Assert.assertFalse(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    File v3TempDir = new SegmentV1V2ToV3FormatConverter().v3ConversionTempDirectory(segmentDirectory);
    FileUtils.touch(v3TempDir);
    {
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
    {
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
        Assert.assertEquals(SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()), SegmentVersion.v3);
        Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
        Assert.assertFalse(v3TempDir.exists());
    }
}

Example 4 with SegmentV1V2ToV3FormatConverter

use of com.linkedin.pinot.core.segment.index.converter.SegmentV1V2ToV3FormatConverter in project pinot by linkedin.

the class SegmentPreProcessorTest method testV3CreateInvertedIndices.

@Test
public void testV3CreateInvertedIndices() throws Exception {
    constructSegment();
    // Convert segment format to v3.
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(_segmentDirectoryFile);
    File v3SegmentDirectoryFile = new File(_segmentDirectoryFile, V3_SEGMENT_NAME);
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(v3SegmentDirectoryFile);
    String segmentVersion = segmentMetadata.getVersion();
    Assert.assertEquals(SegmentVersion.valueOf(segmentVersion), SegmentVersion.v3);
    File singleFileIndex = new File(v3SegmentDirectoryFile, "columns.psf");
    FileTime lastModifiedTime = Files.getLastModifiedTime(singleFileIndex.toPath());
    long fileSize = singleFileIndex.length();
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the first time.
    checkInvertedIndexCreation(v3SegmentDirectoryFile, segmentMetadata, false);
    long addedLength = 0L;
    try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(v3SegmentDirectoryFile, segmentMetadata, ReadMode.mmap);
        SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
        // 8 bytes overhead is for checking integrity of the segment.
        try (PinotDataBuffer col1Buffer = reader.getIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX)) {
            addedLength += col1Buffer.size() + 8;
        }
        try (PinotDataBuffer col13Buffer = reader.getIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX)) {
            addedLength += col13Buffer.size() + 8;
        }
    }
    FileTime newLastModifiedTime = Files.getLastModifiedTime(singleFileIndex.toPath());
    Assert.assertTrue(newLastModifiedTime.compareTo(lastModifiedTime) > 0);
    long newFileSize = singleFileIndex.length();
    Assert.assertEquals(fileSize + addedLength, newFileSize);
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the second time.
    checkInvertedIndexCreation(v3SegmentDirectoryFile, segmentMetadata, true);
    Assert.assertEquals(Files.getLastModifiedTime(singleFileIndex.toPath()), newLastModifiedTime);
    Assert.assertEquals(singleFileIndex.length(), newFileSize);
}