Examples with SegmentVersion com.linkedin.pinot.core.indexsegment.generator.SegmentVersion used on opensource projects

Example 1 with SegmentVersion

use of com.linkedin.pinot.core.indexsegment.generator.SegmentVersion in project pinot by linkedin.

the class TableDataManagerConfig method overrideConfigs.

public void overrideConfigs(String tableName, AbstractTableConfig tableConfig) {
    IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
    _tableDataManagerConfig.setProperty(READ_MODE, indexingConfig.getLoadMode().toLowerCase());
    _tableDataManagerConfig.setProperty(TABLE_DATA_MANAGER_NAME, tableConfig.getTableName());
    List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns();
    if (invertedIndexColumns != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, invertedIndexColumns);
    }
    String starTreeFormat = indexingConfig.getStarTreeFormat();
    if (starTreeFormat != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_STAR_TREE_FORMAT_VERSION, starTreeFormat);
    }
    String columnMinMaxValueGeneratorMode = indexingConfig.getColumnMinMaxValueGeneratorMode();
    if (columnMinMaxValueGeneratorMode != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_COLUMN_MIN_MAX_VALUE_GENERATOR_MODE, columnMinMaxValueGeneratorMode);
    }
    // Server configuration is always to DEFAULT or configured value
    // Apply table configuration only if the server configuration is set with table config
    // overriding server config
    //
    // Server config not set means table config has no impact. This provides additional
    // security from inadvertent changes as both config will need to be enabled for the
    // change to take effect
    SegmentVersion tableConfigVersion = SegmentVersion.fromString(indexingConfig.getSegmentFormatVersion(), SegmentVersion.DEFAULT_TABLE_VERSION);
    SegmentVersion serverConfigVersion = SegmentVersion.fromString(_tableDataManagerConfig.getString(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION), SegmentVersion.DEFAULT_SERVER_VERSION);
    // is less than server configuration
    if (SegmentVersion.compare(tableConfigVersion, serverConfigVersion) < 0) {
        LOGGER.info("Overriding server segment format version: {} with table version: {} for table: {}", serverConfigVersion, tableConfigVersion, tableName);
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION, indexingConfig.getSegmentFormatVersion());
    } else {
        LOGGER.info("Loading table: {} with server configured segment format version: {}", tableName, serverConfigVersion);
    }
}

Example 2 with SegmentVersion

use of com.linkedin.pinot.core.indexsegment.generator.SegmentVersion in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverter method convert.

// NOTE: this can convert segments in v1 and v2 format to v3.
// we use variable names with v2 prefix for readability
@Override
public void convert(File v2SegmentDirectory) throws Exception {
    Preconditions.checkNotNull(v2SegmentDirectory, "Segment directory should not be null");
    Preconditions.checkState(v2SegmentDirectory.exists() && v2SegmentDirectory.isDirectory(), "Segment directory: " + v2SegmentDirectory.toString() + " must exist and should be a directory");
    LOGGER.info("Converting segment: {} to v3 format", v2SegmentDirectory);
    // check existing segment version
    SegmentMetadataImpl v2Metadata = new SegmentMetadataImpl(v2SegmentDirectory);
    SegmentVersion oldVersion = SegmentVersion.valueOf(v2Metadata.getVersion());
    Preconditions.checkState(oldVersion != SegmentVersion.v3, "Segment {} is already in v3 format but at wrong path", v2Metadata.getName());
    Preconditions.checkArgument(oldVersion == SegmentVersion.v1 || oldVersion == SegmentVersion.v2, "Can not convert segment version: {} at path: {} ", oldVersion, v2SegmentDirectory);
    deleteStaleConversionDirectories(v2SegmentDirectory);
    File v3TempDirectory = v3ConversionTempDirectory(v2SegmentDirectory);
    setDirectoryPermissions(v3TempDirectory);
    createMetadataFile(v2SegmentDirectory, v3TempDirectory);
    copyCreationMetadata(v2SegmentDirectory, v3TempDirectory);
    copyIndexData(v2SegmentDirectory, v2Metadata, v3TempDirectory);
    File newLocation = SegmentDirectoryPaths.segmentDirectoryFor(v2SegmentDirectory, SegmentVersion.v3);
    LOGGER.info("v3 segment location for segment: {} is {}", v2Metadata.getName(), newLocation);
    v3TempDirectory.renameTo(newLocation);
    deleteV2Files(v2SegmentDirectory);
}

Example 3 with SegmentVersion

use of com.linkedin.pinot.core.indexsegment.generator.SegmentVersion in project pinot by linkedin.

the class SegmentVersionTest method test1.

@Test
public void test1() {
    SegmentVersion version;
    version = SegmentVersion.fromString("v1", SegmentVersion.v3);
    Assert.assertEquals(version, SegmentVersion.v1);
    version = SegmentVersion.fromString("v2", SegmentVersion.v1);
    Assert.assertEquals(version, SegmentVersion.v2);
    version = SegmentVersion.fromString("v3", SegmentVersion.v1);
    Assert.assertEquals(version, SegmentVersion.v3);
    version = SegmentVersion.fromString("badString", SegmentVersion.v1);
    Assert.assertEquals(version, SegmentVersion.v1);
    version = SegmentVersion.fromString(null, SegmentVersion.v3);
    Assert.assertEquals(version, SegmentVersion.v3);
    version = SegmentVersion.fromString("", SegmentVersion.DEFAULT_SERVER_VERSION);
    Assert.assertEquals(version, SegmentVersion.DEFAULT_SERVER_VERSION);
    Assert.assertTrue(SegmentVersion.v1.compareTo(SegmentVersion.v2) < 0);
    Assert.assertTrue(SegmentVersion.v2.compareTo(SegmentVersion.v2) == 0);
    Assert.assertTrue(SegmentVersion.v3.compareTo(SegmentVersion.v2) > 0);
}

Example 4 with SegmentVersion

use of com.linkedin.pinot.core.indexsegment.generator.SegmentVersion in project pinot by linkedin.

the class SegmentLocalFSDirectory method loadData.

private synchronized void loadData() throws IOException {
    if (columnIndexDirectory != null) {
        return;
    }
    String version = segmentMetadata.getVersion();
    SegmentVersion segmentVersion = SegmentVersion.valueOf(version);
    switch(segmentVersion) {
        case v1:
        case v2:
            columnIndexDirectory = new FilePerIndexDirectory(segmentDirectory, segmentMetadata, readMode);
            break;
        case v3:
            try {
                columnIndexDirectory = new SingleFileIndexDirectory(segmentDirectory, segmentMetadata, readMode);
            } catch (ConfigurationException e) {
                LOGGER.error("Failed to create columnar index directory", e);
                throw new RuntimeException(e);
            }
            break;
    }
}

Example 5 with SegmentVersion

use of com.linkedin.pinot.core.indexsegment.generator.SegmentVersion in project pinot by linkedin.

the class UploadRefreshDeleteIntegrationTest method setupMethod.

@BeforeMethod
public void setupMethod(Object[] args) throws Exception {
    ensureDirectoryExistsAndIsEmpty(_tmpDir);
    ensureDirectoryExistsAndIsEmpty(_segmentsDir);
    ensureDirectoryExistsAndIsEmpty(_tarsDir);
    if (args == null || args.length == 0) {
        return;
    }
    this.tableName = (String) args[0];
    SegmentVersion version = (SegmentVersion) args[1];
    addOfflineTable("DaysSinceEpoch", "daysSinceEpoch", -1, "", null, null, this.tableName, version);
}