Examples with IndexingConfig com.linkedin.pinot.common.config.IndexingConfig used on opensource projects

Example 1 with IndexingConfig

use of com.linkedin.pinot.common.config.IndexingConfig in project pinot by linkedin.

the class PinotHelixResourceManager method addTable.

/**
   * Table APIs
   */
public void addTable(AbstractTableConfig config) throws JsonGenerationException, JsonMappingException, IOException {
    TenantConfig tenantConfig = null;
    TableType type = TableType.valueOf(config.getTableType().toUpperCase());
    if (isSingleTenantCluster()) {
        tenantConfig = new TenantConfig();
        tenantConfig.setBroker(ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
        switch(type) {
            case OFFLINE:
                tenantConfig.setServer(ControllerTenantNameBuilder.getOfflineTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            case REALTIME:
                tenantConfig.setServer(ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            default:
                throw new RuntimeException("UnSupported table type");
        }
        config.setTenantConfig(tenantConfig);
    } else {
        tenantConfig = config.getTenantConfig();
        if (tenantConfig.getBroker() == null || tenantConfig.getServer() == null) {
            throw new RuntimeException("missing tenant configs");
        }
    }
    SegmentsValidationAndRetentionConfig segmentsConfig = config.getValidationConfig();
    switch(type) {
        case OFFLINE:
            final String offlineTableName = config.getTableName();
            // now lets build an ideal state
            LOGGER.info("building empty ideal state for table : " + offlineTableName);
            final IdealState offlineIdealState = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(offlineTableName, Integer.parseInt(segmentsConfig.getReplication()));
            LOGGER.info("adding table via the admin");
            _helixAdmin.addResource(_helixClusterName, offlineTableName, offlineIdealState);
            LOGGER.info("successfully added the table : " + offlineTableName + " to the cluster");
            // lets add table configs
            ZKMetadataProvider.setOfflineTableConfig(_propertyStore, offlineTableName, AbstractTableConfig.toZnRecord(config));
            _propertyStore.create(ZKMetadataProvider.constructPropertyStorePathForResource(offlineTableName), new ZNRecord(offlineTableName), AccessOption.PERSISTENT);
            break;
        case REALTIME:
            final String realtimeTableName = config.getTableName();
            // lets add table configs
            ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, realtimeTableName, AbstractTableConfig.toZnRecord(config));
            /*
         * PinotRealtimeSegmentManager sets up watches on table and segment path. When a table gets created,
         * it expects the INSTANCE path in propertystore to be set up so that it can get the kafka group ID and
         * create (high-level consumer) segments for that table.
         * So, we need to set up the instance first, before adding the table resource for HLC new table creation.
         *
         * For low-level consumers, the order is to create the resource first, and set up the propertystore with segments
         * and then tweak the idealstate to add those segments.
         *
         * We also need to support the case when a high-level consumer already exists for a table and we are adding
         * the low-level consumers.
         */
            IndexingConfig indexingConfig = config.getIndexingConfig();
            ensureRealtimeClusterIsSetUp(config, realtimeTableName, indexingConfig);
            LOGGER.info("Successfully added or updated the table {} ", realtimeTableName);
            break;
        default:
            throw new RuntimeException("UnSupported table type");
    }
    handleBrokerResource(config);
}

Example 2 with IndexingConfig

use of com.linkedin.pinot.common.config.IndexingConfig in project pinot by linkedin.

the class AutoAddInvertedIndex method runQueryStrategy.

private void runQueryStrategy() throws Exception {
    // Get all resources in cluster
    List<String> resourcesInCluster = _helixAdmin.getResourcesInCluster(_clusterName);
    for (String tableName : resourcesInCluster) {
        // Skip non-table resources
        if (!tableName.endsWith("_OFFLINE") && !tableName.endsWith("_REALTIME")) {
            continue;
        }
        // Skip tables that do not match the defined name pattern
        if (_tableNamePattern != null && !tableName.matches(_tableNamePattern)) {
            continue;
        }
        LOGGER.info("Table: {} matches the table name pattern: {}", tableName, _tableNamePattern);
        // Get the inverted index config
        AbstractTableConfig tableConfig = getTableConfig(tableName);
        IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
        List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns();
        boolean autoGeneratedInvertedIndex = indexingConfig.isAutoGeneratedInvertedIndex();
        // Handle auto-generated inverted index
        if (autoGeneratedInvertedIndex) {
            Preconditions.checkState(!invertedIndexColumns.isEmpty(), "Auto-generated inverted index list is empty");
            // NEW mode, skip
            if (_mode == Mode.NEW) {
                LOGGER.info("Table: {}, skip adding inverted index because it has auto-generated inverted index and under NEW mode", tableName);
                continue;
            }
            // REMOVE mode, remove the inverted index and update
            if (_mode == Mode.REMOVE) {
                invertedIndexColumns.clear();
                indexingConfig.setAutoGeneratedInvertedIndex(false);
                if (updateIndexConfig(tableName, tableConfig)) {
                    LOGGER.info("Table: {}, removed auto-generated inverted index", tableName);
                } else {
                    LOGGER.error("Table: {}, failed to remove auto-generated inverted index", tableName);
                }
                continue;
            }
            // REFRESH mode, remove auto-generated inverted index
            if (_mode == Mode.REFRESH) {
                invertedIndexColumns.clear();
            }
        } else {
            // Handle null inverted index columns
            if (invertedIndexColumns == null) {
                invertedIndexColumns = new ArrayList<>();
                indexingConfig.setInvertedIndexColumns(invertedIndexColumns);
            }
            // Remove empty strings
            int emptyStringIndex;
            while ((emptyStringIndex = invertedIndexColumns.indexOf("")) != -1) {
                invertedIndexColumns.remove(emptyStringIndex);
            }
            // Skip non-empty non-auto-generated inverted index
            if (!invertedIndexColumns.isEmpty()) {
                LOGGER.info("Table: {}, skip adding inverted index because it has non-auto-generated inverted index", tableName);
                continue;
            }
        }
        // Skip tables without a schema
        Schema tableSchema = getTableSchema(tableName);
        if (tableSchema == null) {
            LOGGER.info("Table: {}, skip adding inverted index because it does not have a schema", tableName);
            continue;
        }
        // Skip tables without dimensions
        List<String> dimensionNames = tableSchema.getDimensionNames();
        if (dimensionNames.size() == 0) {
            LOGGER.info("Table: {}, skip adding inverted index because it does not have any dimension column", tableName);
            continue;
        }
        // Skip tables without a proper time column
        TimeFieldSpec timeFieldSpec = tableSchema.getTimeFieldSpec();
        if (timeFieldSpec == null || timeFieldSpec.getDataType() == FieldSpec.DataType.STRING) {
            LOGGER.info("Table: {}, skip adding inverted index because it does not have a numeric time column", tableName);
            continue;
        }
        String timeColumnName = timeFieldSpec.getName();
        TimeUnit timeUnit = timeFieldSpec.getOutgoingGranularitySpec().getTimeType();
        if (timeUnit != TimeUnit.DAYS) {
            LOGGER.warn("Table: {}, time column {] has non-DAYS time unit: {}", timeColumnName, timeUnit);
        }
        // Only add inverted index to table larger than a threshold
        JSONObject queryResponse = sendQuery("SELECT COUNT(*) FROM " + tableName);
        long numTotalDocs = queryResponse.getLong("totalDocs");
        LOGGER.info("Table: {}, number of total documents: {}", tableName, numTotalDocs);
        if (numTotalDocs <= _tableSizeThreshold) {
            LOGGER.info("Table: {}, skip adding inverted index because the table is too small", tableName);
            continue;
        }
        // Get each dimension's cardinality on one timestamp's data
        queryResponse = sendQuery("SELECT Max(" + timeColumnName + ") FROM " + tableName);
        int maxTimeStamp = queryResponse.getJSONArray("aggregationResults").getJSONObject(0).getInt("value");
        LOGGER.info("Table: {}, max time column {}: {}", tableName, timeColumnName, maxTimeStamp);
        // Query DISTINCTCOUNT on all dimensions in one query might cause timeout, so query them separately
        List<ResultPair> resultPairs = new ArrayList<>();
        for (String dimensionName : dimensionNames) {
            String query = "SELECT DISTINCTCOUNT(" + dimensionName + ") FROM " + tableName + " WHERE " + timeColumnName + " = " + maxTimeStamp;
            queryResponse = sendQuery(query);
            JSONObject result = queryResponse.getJSONArray("aggregationResults").getJSONObject(0);
            resultPairs.add(new ResultPair(result.getString("function").substring("distinctCount_".length()), result.getLong("value")));
        }
        // Sort the dimensions based on their cardinalities
        Collections.sort(resultPairs);
        // Add the top dimensions into inverted index columns
        int numInvertedIndex = Math.min(_maxNumInvertedIndexAdded, resultPairs.size());
        for (int i = 0; i < numInvertedIndex; i++) {
            ResultPair resultPair = resultPairs.get(i);
            String columnName = resultPair._key;
            long cardinality = resultPair._value;
            if (cardinality > _cardinalityThreshold) {
                // Do not append inverted index if already exists
                if (!invertedIndexColumns.contains(columnName)) {
                    invertedIndexColumns.add(columnName);
                }
                LOGGER.info("Table: {}, add inverted index to column {} with cardinality: {}", tableName, columnName, cardinality);
            } else {
                LOGGER.info("Table: {}, skip adding inverted index to column {} with cardinality: {}", tableName, columnName, cardinality);
                break;
            }
        }
        // Update indexing config
        if (!invertedIndexColumns.isEmpty()) {
            indexingConfig.setAutoGeneratedInvertedIndex(true);
            if (updateIndexConfig(tableName, tableConfig)) {
                LOGGER.info("Table: {}, added inverted index to columns: {}", tableName, invertedIndexColumns);
            } else {
                LOGGER.error("Table: {}, failed to add inverted index to columns: {}", tableName, invertedIndexColumns);
            }
        } else {
            if (autoGeneratedInvertedIndex) {
                Preconditions.checkState(_mode == Mode.REFRESH);
                // Remove existing auto-generated inverted index because no column matches all the conditions
                indexingConfig.setAutoGeneratedInvertedIndex(false);
                if (updateIndexConfig(tableName, tableConfig)) {
                    LOGGER.info("Table: {}, removed auto-generated inverted index", tableName);
                } else {
                    LOGGER.error("Table: {}, failed to remove auto-generated inverted index", tableName);
                }
            }
        }
    }
}

Example 3 with IndexingConfig

use of com.linkedin.pinot.common.config.IndexingConfig in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManagerTest method testAutoReplaceConsumingSegment.

public void testAutoReplaceConsumingSegment(final String tableConfigStartOffset) throws Exception {
    FakePinotLLCRealtimeSegmentManager segmentManager = new FakePinotLLCRealtimeSegmentManager(true, null);
    final int nPartitions = 8;
    final int nInstances = 3;
    final int nReplicas = 2;
    final String topic = "someTopic";
    final String rtTableName = "table_REALTIME";
    List<String> instances = getInstanceList(nInstances);
    final String startOffset = KAFKA_OFFSET;
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyKafkaConsumerRealtimeIdealStateFor(rtTableName, nReplicas);
    segmentManager.setupHelixEntries(topic, rtTableName, nPartitions, instances, nReplicas, startOffset, DUMMY_HOST, idealState, false, 10000);
    // Add another segment for each partition
    long now = System.currentTimeMillis();
    List<String> existingSegments = new ArrayList<>(segmentManager._idealStateEntries.keySet());
    final int partitionToBeFixed = 3;
    final int partitionWithHigherOffset = 4;
    final int emptyPartition = 5;
    final long smallestPartitionOffset = 0x259080984568L;
    final long largestPartitionOffset = smallestPartitionOffset + 100000;
    final long higherOffset = smallestPartitionOffset + 100;
    for (String segmentNameStr : existingSegments) {
        LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
        switch(segmentName.getPartitionId()) {
            case partitionToBeFixed:
                // Do nothing, we will test adding a new segment for this partition when there is only one segment in there.
                break;
            case emptyPartition:
                // Remove existing segment, so we can test adding a new segment for this partition when none exists
                segmentManager._idealStateEntries.remove(segmentNameStr);
                break;
            case partitionWithHigherOffset:
                // Set segment metadata for this segment such that its offset is higher than startOffset we get from kafka.
                // In that case, we should choose the new segment offset as this one rather than the one kafka hands us.
                LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata();
                metadata.setSegmentName(segmentName.getSegmentName());
                metadata.setEndOffset(higherOffset);
                metadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
                segmentManager._metadataMap.put(segmentName.getSegmentName(), metadata);
                break;
            default:
                // Add a second segment for this partition. It will not be repaired.
                LLCSegmentName newSegmentName = new LLCSegmentName(segmentName.getTableName(), segmentName.getPartitionId(), segmentName.getSequenceNumber() + 1, now);
                List<String> hosts = segmentManager._idealStateEntries.get(segmentNameStr);
                segmentManager._idealStateEntries.put(newSegmentName.getSegmentName(), hosts);
                break;
        }
    }
    Map<String, String> streamPropMap = new HashMap<>(1);
    streamPropMap.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.CONSUMER_TYPE), "simple");
    streamPropMap.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.KAFKA_CONSUMER_PROPS_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.AUTO_OFFSET_RESET), tableConfigStartOffset);
    KafkaStreamMetadata kafkaStreamMetadata = new KafkaStreamMetadata(streamPropMap);
    AbstractTableConfig tableConfig = mock(AbstractTableConfig.class);
    IndexingConfig indexingConfig = mock(IndexingConfig.class);
    when(indexingConfig.getStreamConfigs()).thenReturn(streamPropMap);
    when(tableConfig.getIndexingConfig()).thenReturn(indexingConfig);
    Set<Integer> nonConsumingPartitions = new HashSet<>(1);
    nonConsumingPartitions.add(partitionToBeFixed);
    nonConsumingPartitions.add(partitionWithHigherOffset);
    nonConsumingPartitions.add(emptyPartition);
    segmentManager._kafkaSmallestOffsetToReturn = smallestPartitionOffset;
    segmentManager._kafkaLargestOffsetToReturn = largestPartitionOffset;
    existingSegments = new ArrayList<>(segmentManager._idealStateEntries.keySet());
    segmentManager._paths.clear();
    segmentManager._records.clear();
    segmentManager.createConsumingSegment(rtTableName, nonConsumingPartitions, existingSegments, tableConfig);
    Assert.assertEquals(segmentManager._paths.size(), 3);
    Assert.assertEquals(segmentManager._records.size(), 3);
    Assert.assertEquals(segmentManager._oldSegmentNameStr.size(), 3);
    Assert.assertEquals(segmentManager._newSegmentNameStr.size(), 3);
    int found = 0;
    int index = 0;
    while (index < segmentManager._paths.size()) {
        String znodePath = segmentManager._paths.get(index);
        int slash = znodePath.lastIndexOf('/');
        String segmentNameStr = znodePath.substring(slash + 1);
        LLCSegmentName segmentName = new LLCSegmentName(segmentNameStr);
        ZNRecord znRecord;
        LLCRealtimeSegmentZKMetadata metadata;
        switch(segmentName.getPartitionId()) {
            case partitionToBeFixed:
                // We had left this partition with one segment. So, a second one should be created with a sequence number one
                // higher than starting. Its start offset should be what kafka returns.
                found++;
                Assert.assertEquals(segmentName.getSequenceNumber(), PinotLLCRealtimeSegmentManager.STARTING_SEQUENCE_NUMBER + 1);
                znRecord = segmentManager._records.get(index);
                metadata = new LLCRealtimeSegmentZKMetadata(znRecord);
                Assert.assertEquals(metadata.getNumReplicas(), 2);
                Assert.assertEquals(metadata.getStartOffset(), smallestPartitionOffset);
                break;
            case emptyPartition:
                // We had removed any segments in this partition. A new one should be created with the offset as returned
                // by kafka and with the starting sequence number.
                found++;
                Assert.assertEquals(segmentName.getSequenceNumber(), PinotLLCRealtimeSegmentManager.STARTING_SEQUENCE_NUMBER);
                znRecord = segmentManager._records.get(index);
                metadata = new LLCRealtimeSegmentZKMetadata(znRecord);
                Assert.assertEquals(metadata.getNumReplicas(), 2);
                if (tableConfigStartOffset.equals("smallest")) {
                    Assert.assertEquals(metadata.getStartOffset(), smallestPartitionOffset);
                } else {
                    Assert.assertEquals(metadata.getStartOffset(), largestPartitionOffset);
                }
                break;
            case partitionWithHigherOffset:
                // We had left this partition with one segment. In addition, we had the end-offset of the first segment set to
                // a value higher than that returned by kafka. So, a second one should be created with a sequence number one
                // equal to the end offset of the first one.
                found++;
                Assert.assertEquals(segmentName.getSequenceNumber(), PinotLLCRealtimeSegmentManager.STARTING_SEQUENCE_NUMBER + 1);
                znRecord = segmentManager._records.get(index);
                metadata = new LLCRealtimeSegmentZKMetadata(znRecord);
                Assert.assertEquals(metadata.getNumReplicas(), 2);
                Assert.assertEquals(metadata.getStartOffset(), higherOffset);
                break;
        }
        index++;
    }
    // We should see all three cases here.
    Assert.assertEquals(3, found);
    // Now, if we make 'partitionToBeFixed' a non-consuming partition, a second one should get added with the same start offset as
    // as the first one, since the kafka offset to return has not changed.
    Set<Integer> ncPartitions = new HashSet<>(1);
    ncPartitions.add(partitionToBeFixed);
    segmentManager.createConsumingSegment(rtTableName, ncPartitions, segmentManager.getExistingSegments(rtTableName), tableConfig);
    Assert.assertEquals(segmentManager._paths.size(), 4);
    Assert.assertEquals(segmentManager._records.size(), 4);
    Assert.assertEquals(segmentManager._oldSegmentNameStr.size(), 4);
    Assert.assertEquals(segmentManager._newSegmentNameStr.size(), 4);
    // The latest zn record should be that of the new one we added.
    ZNRecord znRecord = segmentManager._records.get(3);
    LLCRealtimeSegmentZKMetadata metadata = new LLCRealtimeSegmentZKMetadata(znRecord);
    Assert.assertEquals(metadata.getNumReplicas(), 2);
    Assert.assertEquals(metadata.getStartOffset(), smallestPartitionOffset);
    LLCSegmentName llcSegmentName = new LLCSegmentName(metadata.getSegmentName());
    Assert.assertEquals(llcSegmentName.getSequenceNumber(), PinotLLCRealtimeSegmentManager.STARTING_SEQUENCE_NUMBER + 2);
    Assert.assertEquals(llcSegmentName.getPartitionId(), partitionToBeFixed);
    // Now pretend the prev segment ended successfully, and set the end offset
    metadata.setEndOffset(metadata.getStartOffset() + 10);
    metadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
    segmentManager._records.remove(3);
    segmentManager._records.add(metadata.toZNRecord());
    segmentManager._metadataMap.put(metadata.getSegmentName(), metadata);
    segmentManager._kafkaLargestOffsetToReturn *= 2;
    segmentManager._kafkaSmallestOffsetToReturn *= 2;
    ncPartitions.clear();
    ncPartitions.add(partitionToBeFixed);
    segmentManager.createConsumingSegment(rtTableName, ncPartitions, segmentManager.getExistingSegments(rtTableName), tableConfig);
    Assert.assertEquals(segmentManager._paths.size(), 5);
    Assert.assertEquals(segmentManager._records.size(), 5);
    Assert.assertEquals(segmentManager._oldSegmentNameStr.size(), 5);
    Assert.assertEquals(segmentManager._newSegmentNameStr.size(), 5);
    znRecord = segmentManager._records.get(4);
    metadata = new LLCRealtimeSegmentZKMetadata(znRecord);
    Assert.assertEquals(metadata.getNumReplicas(), 2);
    // In this case, since we have data loss, we will always put the smallest kafka partition available.
    Assert.assertEquals(metadata.getStartOffset(), segmentManager.getKafkaPartitionOffset(null, "smallest", partitionToBeFixed));
    llcSegmentName = new LLCSegmentName(metadata.getSegmentName());
    Assert.assertEquals(llcSegmentName.getSequenceNumber(), PinotLLCRealtimeSegmentManager.STARTING_SEQUENCE_NUMBER + 3);
    Assert.assertEquals(llcSegmentName.getPartitionId(), partitionToBeFixed);
}

Example 4 with IndexingConfig

use of com.linkedin.pinot.common.config.IndexingConfig in project pinot by linkedin.

the class ValidationManagerTest method testLLCValidation.

@Test
public void testLLCValidation() throws Exception {
    final String topicName = "topic";
    final int kafkaPartitionCount = 2;
    final String realtimeTableName = "table_REALTIME";
    final String tableName = TableNameBuilder.extractRawTableName(realtimeTableName);
    // Server 1
    final String S1 = "S1";
    // Server 2
    final String S2 = "S2";
    // Server 3
    final String S3 = "S3";
    final List<String> hosts = Arrays.asList(new String[] { S1, S2, S3 });
    final HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    ZNRecord znRecord = new ZNRecord(topicName);
    for (int i = 0; i < kafkaPartitionCount; i++) {
        znRecord.setListField(Integer.toString(i), hosts);
    }
    makeMockPinotLLCRealtimeSegmentManager(znRecord);
    long msSinceEpoch = 1540;
    LLCSegmentName p0s0 = new LLCSegmentName(tableName, 0, 0, msSinceEpoch);
    LLCSegmentName p0s1 = new LLCSegmentName(tableName, 0, 1, msSinceEpoch);
    LLCSegmentName p1s0 = new LLCSegmentName(tableName, 1, 0, msSinceEpoch);
    LLCSegmentName p1s1 = new LLCSegmentName(tableName, 1, 1, msSinceEpoch);
    IdealState idealstate = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(realtimeTableName, 3);
    idealstate.setPartitionState(p0s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p0s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    //    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s0.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p1s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    FakeValidationMetrics validationMetrics = new FakeValidationMetrics();
    ValidationManager validationManager = new ValidationManager(validationMetrics, _pinotHelixResourceManager, new ControllerConf(), _segmentManager);
    Map<String, String> streamConfigs = new HashMap<String, String>(4);
    streamConfigs.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.CONSUMER_TYPE), "highLevel,simple");
    Field autoCreateOnError = ValidationManager.class.getDeclaredField("_autoCreateOnError");
    autoCreateOnError.setAccessible(true);
    autoCreateOnError.setBoolean(validationManager, false);
    AbstractTableConfig tableConfig = mock(AbstractTableConfig.class);
    IndexingConfig indexingConfig = mock(IndexingConfig.class);
    when(tableConfig.getIndexingConfig()).thenReturn(indexingConfig);
    when(indexingConfig.getStreamConfigs()).thenReturn(streamConfigs);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 1);
    // Set partition 0 to have one instance in CONSUMING state, and others in OFFLINE.
    // we should not flag any partitions to correct.
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
    idealstate.setPartitionState(p0s1.getSegmentName(), S1, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S2, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    idealstate.setPartitionState(p0s1.getSegmentName(), S3, PinotHelixSegmentOnlineOfflineStateModelGenerator.OFFLINE_STATE);
    helixAdmin.addResource(HELIX_CLUSTER_NAME, realtimeTableName, idealstate);
    validationManager.validateLLCSegments(realtimeTableName, tableConfig);
    Assert.assertEquals(validationMetrics.partitionCount, 0);
    helixAdmin.dropResource(HELIX_CLUSTER_NAME, realtimeTableName);
}

Example 5 with IndexingConfig

use of com.linkedin.pinot.common.config.IndexingConfig in project pinot by linkedin.

the class TableDataManagerConfig method overrideConfigs.

public void overrideConfigs(String tableName, AbstractTableConfig tableConfig) {
    IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
    _tableDataManagerConfig.setProperty(READ_MODE, indexingConfig.getLoadMode().toLowerCase());
    _tableDataManagerConfig.setProperty(TABLE_DATA_MANAGER_NAME, tableConfig.getTableName());
    List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns();
    if (invertedIndexColumns != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_LOADING_INVERTED_INDEX, invertedIndexColumns);
    }
    String starTreeFormat = indexingConfig.getStarTreeFormat();
    if (starTreeFormat != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_STAR_TREE_FORMAT_VERSION, starTreeFormat);
    }
    String columnMinMaxValueGeneratorMode = indexingConfig.getColumnMinMaxValueGeneratorMode();
    if (columnMinMaxValueGeneratorMode != null) {
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_COLUMN_MIN_MAX_VALUE_GENERATOR_MODE, columnMinMaxValueGeneratorMode);
    }
    // Server configuration is always to DEFAULT or configured value
    // Apply table configuration only if the server configuration is set with table config
    // overriding server config
    //
    // Server config not set means table config has no impact. This provides additional
    // security from inadvertent changes as both config will need to be enabled for the
    // change to take effect
    SegmentVersion tableConfigVersion = SegmentVersion.fromString(indexingConfig.getSegmentFormatVersion(), SegmentVersion.DEFAULT_TABLE_VERSION);
    SegmentVersion serverConfigVersion = SegmentVersion.fromString(_tableDataManagerConfig.getString(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION), SegmentVersion.DEFAULT_SERVER_VERSION);
    // is less than server configuration
    if (SegmentVersion.compare(tableConfigVersion, serverConfigVersion) < 0) {
        LOGGER.info("Overriding server segment format version: {} with table version: {} for table: {}", serverConfigVersion, tableConfigVersion, tableName);
        _tableDataManagerConfig.setProperty(IndexLoadingConfigMetadata.KEY_OF_SEGMENT_FORMAT_VERSION, indexingConfig.getSegmentFormatVersion());
    } else {
        LOGGER.info("Loading table: {} with server configured segment format version: {}", tableName, serverConfigVersion);
    }
}