Examples with TenantConfig com.linkedin.pinot.common.config.TenantConfig used on opensource projects

Example 1 with TenantConfig

use of com.linkedin.pinot.common.config.TenantConfig in project pinot by linkedin.

the class PinotHelixResourceManager method rebuildBrokerResourceFromHelixTags.

public PinotResourceManagerResponse rebuildBrokerResourceFromHelixTags(final String tableName) {
    // Get the broker tag for this table
    String brokerTag = null;
    TenantConfig tenantConfig = null;
    try {
        final TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
        AbstractTableConfig tableConfig;
        if (tableType == TableType.OFFLINE) {
            tableConfig = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), tableName);
        } else if (tableType == TableType.REALTIME) {
            tableConfig = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), tableName);
        } else {
            return new PinotResourceManagerResponse("Table " + tableName + " does not have a table type", false);
        }
        if (tableConfig == null) {
            return new PinotResourceManagerResponse("Table " + tableName + " does not exist", false);
        }
        tenantConfig = tableConfig.getTenantConfig();
    } catch (Exception e) {
        LOGGER.warn("Caught exception while getting tenant config for table {}", tableName, e);
        return new PinotResourceManagerResponse("Failed to fetch broker tag for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
    brokerTag = tenantConfig.getBroker();
    // Look for all instances tagged with this broker tag
    final Set<String> brokerInstances = getAllInstancesForBrokerTenant(brokerTag);
    // If we add a new broker, we want to rebuild the broker resource.
    HelixAdmin helixAdmin = getHelixAdmin();
    String clusterName = getHelixClusterName();
    IdealState brokerIdealState = HelixHelper.getBrokerIdealStates(helixAdmin, clusterName);
    Set<String> idealStateBrokerInstances = brokerIdealState.getInstanceSet(tableName);
    if (idealStateBrokerInstances.equals(brokerInstances)) {
        return new PinotResourceManagerResponse("Broker resource is not rebuilt because ideal state is the same for table {} " + tableName, false);
    }
    // Reset ideal state with the instance list
    try {
        HelixHelper.updateIdealState(getHelixZkManager(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, new Function<IdealState, IdealState>() {

            @Nullable
            @Override
            public IdealState apply(@Nullable IdealState idealState) {
                Map<String, String> instanceStateMap = idealState.getInstanceStateMap(tableName);
                if (instanceStateMap != null) {
                    instanceStateMap.clear();
                }
                for (String brokerInstance : brokerInstances) {
                    idealState.setPartitionState(tableName, brokerInstance, BrokerOnlineOfflineStateModel.ONLINE);
                }
                return idealState;
            }
        }, DEFAULT_RETRY_POLICY);
        LOGGER.info("Successfully rebuilt brokerResource for table {}", tableName);
        return new PinotResourceManagerResponse("Rebuilt brokerResource for table " + tableName, true);
    } catch (Exception e) {
        LOGGER.warn("Caught exception while rebuilding broker resource from Helix tags for table {}", e, tableName);
        return new PinotResourceManagerResponse("Failed to rebuild brokerResource for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
}

Example 2 with TenantConfig

use of com.linkedin.pinot.common.config.TenantConfig in project pinot by linkedin.

the class PinotHelixResourceManager method addTable.

/**
   * Table APIs
   */
public void addTable(AbstractTableConfig config) throws JsonGenerationException, JsonMappingException, IOException {
    TenantConfig tenantConfig = null;
    TableType type = TableType.valueOf(config.getTableType().toUpperCase());
    if (isSingleTenantCluster()) {
        tenantConfig = new TenantConfig();
        tenantConfig.setBroker(ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
        switch(type) {
            case OFFLINE:
                tenantConfig.setServer(ControllerTenantNameBuilder.getOfflineTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            case REALTIME:
                tenantConfig.setServer(ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            default:
                throw new RuntimeException("UnSupported table type");
        }
        config.setTenantConfig(tenantConfig);
    } else {
        tenantConfig = config.getTenantConfig();
        if (tenantConfig.getBroker() == null || tenantConfig.getServer() == null) {
            throw new RuntimeException("missing tenant configs");
        }
    }
    SegmentsValidationAndRetentionConfig segmentsConfig = config.getValidationConfig();
    switch(type) {
        case OFFLINE:
            final String offlineTableName = config.getTableName();
            // now lets build an ideal state
            LOGGER.info("building empty ideal state for table : " + offlineTableName);
            final IdealState offlineIdealState = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(offlineTableName, Integer.parseInt(segmentsConfig.getReplication()));
            LOGGER.info("adding table via the admin");
            _helixAdmin.addResource(_helixClusterName, offlineTableName, offlineIdealState);
            LOGGER.info("successfully added the table : " + offlineTableName + " to the cluster");
            // lets add table configs
            ZKMetadataProvider.setOfflineTableConfig(_propertyStore, offlineTableName, AbstractTableConfig.toZnRecord(config));
            _propertyStore.create(ZKMetadataProvider.constructPropertyStorePathForResource(offlineTableName), new ZNRecord(offlineTableName), AccessOption.PERSISTENT);
            break;
        case REALTIME:
            final String realtimeTableName = config.getTableName();
            // lets add table configs
            ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, realtimeTableName, AbstractTableConfig.toZnRecord(config));
            /*
         * PinotRealtimeSegmentManager sets up watches on table and segment path. When a table gets created,
         * it expects the INSTANCE path in propertystore to be set up so that it can get the kafka group ID and
         * create (high-level consumer) segments for that table.
         * So, we need to set up the instance first, before adding the table resource for HLC new table creation.
         *
         * For low-level consumers, the order is to create the resource first, and set up the propertystore with segments
         * and then tweak the idealstate to add those segments.
         *
         * We also need to support the case when a high-level consumer already exists for a table and we are adding
         * the low-level consumers.
         */
            IndexingConfig indexingConfig = config.getIndexingConfig();
            ensureRealtimeClusterIsSetUp(config, realtimeTableName, indexingConfig);
            LOGGER.info("Successfully added or updated the table {} ", realtimeTableName);
            break;
        default:
            throw new RuntimeException("UnSupported table type");
    }
    handleBrokerResource(config);
}

Example 3 with TenantConfig

use of com.linkedin.pinot.common.config.TenantConfig in project pinot by linkedin.

the class PinotLLCRealtimeSegmentManagerTest method makeTableConfig.

// Make a tableconfig that returns the topic name and nReplicas per partition as we need it.
private AbstractTableConfig makeTableConfig(int nReplicas, String topic) {
    AbstractTableConfig mockTableConfig = mock(AbstractTableConfig.class);
    SegmentsValidationAndRetentionConfig mockValidationConfig = mock(SegmentsValidationAndRetentionConfig.class);
    when(mockValidationConfig.getReplicasPerPartition()).thenReturn(Integer.toString(nReplicas));
    when(mockTableConfig.getValidationConfig()).thenReturn(mockValidationConfig);
    Map<String, String> streamConfigMap = new HashMap<>(1);
    String consumerTypesCsv = streamConfigMap.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.CONSUMER_TYPE), "simple");
    streamConfigMap.put(StringUtil.join(".", CommonConstants.Helix.DataSource.STREAM_PREFIX, CommonConstants.Helix.DataSource.Realtime.Kafka.TOPIC_NAME), topic);
    IndexingConfig mockIndexConfig = mock(IndexingConfig.class);
    when(mockIndexConfig.getStreamConfigs()).thenReturn(streamConfigMap);
    when(mockTableConfig.getIndexingConfig()).thenReturn(mockIndexConfig);
    TenantConfig mockTenantConfig = mock(TenantConfig.class);
    when(mockTenantConfig.getServer()).thenReturn("freeTenant");
    when(mockTableConfig.getTenantConfig()).thenReturn(mockTenantConfig);
    return mockTableConfig;
}