Examples with SegmentMetadataImpl com.linkedin.pinot.core.segment.index.SegmentMetadataImpl used on opensource projects

Example 6 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class ColumnValueSegmentPruner method prune.

@Override
public boolean prune(@Nonnull IndexSegment segment, @Nonnull BrokerRequest brokerRequest) {
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    if (filterQueryTree == null) {
        return false;
    }
    // For realtime segment, this map can be null.
    Map<String, ColumnMetadata> columnMetadataMap = ((SegmentMetadataImpl) segment.getSegmentMetadata()).getColumnMetadataMap();
    return (columnMetadataMap != null) && pruneSegment(filterQueryTree, columnMetadataMap);
}

Example 7 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverter method convert.

// NOTE: this can convert segments in v1 and v2 format to v3.
// we use variable names with v2 prefix for readability
@Override
public void convert(File v2SegmentDirectory) throws Exception {
    Preconditions.checkNotNull(v2SegmentDirectory, "Segment directory should not be null");
    Preconditions.checkState(v2SegmentDirectory.exists() && v2SegmentDirectory.isDirectory(), "Segment directory: " + v2SegmentDirectory.toString() + " must exist and should be a directory");
    LOGGER.info("Converting segment: {} to v3 format", v2SegmentDirectory);
    // check existing segment version
    SegmentMetadataImpl v2Metadata = new SegmentMetadataImpl(v2SegmentDirectory);
    SegmentVersion oldVersion = SegmentVersion.valueOf(v2Metadata.getVersion());
    Preconditions.checkState(oldVersion != SegmentVersion.v3, "Segment {} is already in v3 format but at wrong path", v2Metadata.getName());
    Preconditions.checkArgument(oldVersion == SegmentVersion.v1 || oldVersion == SegmentVersion.v2, "Can not convert segment version: {} at path: {} ", oldVersion, v2SegmentDirectory);
    deleteStaleConversionDirectories(v2SegmentDirectory);
    File v3TempDirectory = v3ConversionTempDirectory(v2SegmentDirectory);
    setDirectoryPermissions(v3TempDirectory);
    createMetadataFile(v2SegmentDirectory, v3TempDirectory);
    copyCreationMetadata(v2SegmentDirectory, v3TempDirectory);
    copyIndexData(v2SegmentDirectory, v2Metadata, v3TempDirectory);
    File newLocation = SegmentDirectoryPaths.segmentDirectoryFor(v2SegmentDirectory, SegmentVersion.v3);
    LOGGER.info("v3 segment location for segment: {} is {}", v2Metadata.getName(), newLocation);
    v3TempDirectory.renameTo(newLocation);
    deleteV2Files(v2SegmentDirectory);
}

Example 8 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverter method copyIndexData.

private void copyIndexData(File v2Directory, SegmentMetadataImpl v2Metadata, File v3Directory) throws Exception {
    SegmentMetadataImpl v3Metadata = new SegmentMetadataImpl(v3Directory);
    try (SegmentDirectory v2Segment = SegmentDirectory.createFromLocalFS(v2Directory, v2Metadata, ReadMode.mmap);
        SegmentDirectory v3Segment = SegmentDirectory.createFromLocalFS(v3Directory, v3Metadata, ReadMode.mmap)) {
        // for each dictionary and each fwdIndex, copy that to newDirectory buffer
        Set<String> allColumns = v2Metadata.getAllColumns();
        try (SegmentDirectory.Reader v2DataReader = v2Segment.createReader();
            SegmentDirectory.Writer v3DataWriter = v3Segment.createWriter()) {
            for (String column : allColumns) {
                LOGGER.debug("Converting segment: {} , column: {}", v2Directory, column);
                if (v2Metadata.hasDictionary(column)) {
                    copyDictionary(v2DataReader, v3DataWriter, column);
                }
                copyForwardIndex(v2DataReader, v3DataWriter, column);
            }
            // inverted indexes are intentionally stored at the end of the single file
            for (String column : allColumns) {
                copyExistingInvertedIndex(v2DataReader, v3DataWriter, column);
            }
            copyStarTree(v2DataReader, v3DataWriter);
            v3DataWriter.saveAndClose();
        }
    }
}

Example 9 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class HllIndexCreationTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentWithHllIndexCreateHelper helper = null;
    try {
        helper = new SegmentWithHllIndexCreateHelper("testConvert", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment");
        SegmentIndexCreationDriver driver = helper.build(true, hllConfig);
        File segmentDirectory = new File(helper.getIndexDir(), driver.getSegmentName());
        LOGGER.debug("Segment Directory: " + segmentDirectory.getAbsolutePath());
        SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
        converter.convert(segmentDirectory);
        File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
        LOGGER.debug("v3Location: " + v3Location.getAbsolutePath());
        Assert.assertTrue(v3Location.exists());
        Assert.assertTrue(v3Location.isDirectory());
        Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
        SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
        LOGGER.debug("metadata all columns: " + metadata.getAllColumns());
        Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
        // Drop the star tree index file because it has invalid data
        // new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
        // new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
        FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
        // verify that the segment loads correctly. This is necessary and sufficient
        // full proof way to ensure that segment is correctly translated
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
        Assert.assertNotNull(indexSegment);
        Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
        Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    } finally {
        if (helper != null) {
            helper.cleanTempDir();
        }
    }
}

Example 10 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentPreProcessorTest method testV1CreateInvertedIndices.

@Test
public void testV1CreateInvertedIndices() throws Exception {
    constructSegment();
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(_segmentDirectoryFile);
    String segmentVersion = segmentMetadata.getVersion();
    Assert.assertEquals(SegmentVersion.valueOf(segmentVersion), SegmentVersion.v1);
    String col1FileName = segmentMetadata.getBitmapInvertedIndexFileName(COLUMN1_NAME, segmentVersion);
    String col7FileName = segmentMetadata.getBitmapInvertedIndexFileName(COLUMN7_NAME, segmentVersion);
    String col13FileName = segmentMetadata.getBitmapInvertedIndexFileName(COLUMN13_NAME, segmentVersion);
    String badColFileName = segmentMetadata.getBitmapInvertedIndexFileName(NO_SUCH_COLUMN_NAME, segmentVersion);
    File col1File = new File(_segmentDirectoryFile, col1FileName);
    File col7File = new File(_segmentDirectoryFile, col7FileName);
    File col13File = new File(_segmentDirectoryFile, col13FileName);
    File badColFile = new File(_segmentDirectoryFile, badColFileName);
    Assert.assertFalse(col1File.exists());
    Assert.assertTrue(col7File.exists());
    Assert.assertFalse(col13File.exists());
    Assert.assertFalse(badColFile.exists());
    FileTime col7LastModifiedTime = Files.getLastModifiedTime(col7File.toPath());
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the first time.
    checkInvertedIndexCreation(_segmentDirectoryFile, segmentMetadata, false);
    Assert.assertTrue(col1File.exists());
    Assert.assertTrue(col7File.exists());
    Assert.assertTrue(col13File.exists());
    Assert.assertFalse(badColFile.exists());
    Assert.assertEquals(Files.getLastModifiedTime(col7File.toPath()), col7LastModifiedTime);
    // Update inverted index file last modified time.
    FileTime col1LastModifiedTime = Files.getLastModifiedTime(col1File.toPath());
    FileTime col13LastModifiedTime = Files.getLastModifiedTime(col13File.toPath());
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the second time.
    checkInvertedIndexCreation(_segmentDirectoryFile, segmentMetadata, true);
    Assert.assertTrue(col1File.exists());
    Assert.assertTrue(col7File.exists());
    Assert.assertTrue(col13File.exists());
    Assert.assertFalse(badColFile.exists());
    Assert.assertEquals(Files.getLastModifiedTime(col1File.toPath()), col1LastModifiedTime);
    Assert.assertEquals(Files.getLastModifiedTime(col7File.toPath()), col7LastModifiedTime);
    Assert.assertEquals(Files.getLastModifiedTime(col13File.toPath()), col13LastModifiedTime);
}