Examples with AbstractTableConfig com.linkedin.pinot.common.config.AbstractTableConfig used on opensource projects

Example 46 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method getBrokerInstancesForTable.

public List<String> getBrokerInstancesForTable(String tableName, TableType type) throws JsonParseException, JsonMappingException, JsonProcessingException, JSONException, IOException {
    String actualTableName = new TableNameBuilder(type).forTable(tableName);
    AbstractTableConfig config = null;
    if (type == TableType.REALTIME) {
        config = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), actualTableName);
    } else {
        config = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), actualTableName);
    }
    String brokerTenantName = ControllerTenantNameBuilder.getBrokerTenantNameForTenant(config.getTenantConfig().getBroker());
    List<String> serverInstances = _helixAdmin.getInstancesInClusterWithTag(_helixClusterName, brokerTenantName);
    return serverInstances;
}

Example 47 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method getServerInstancesForTable.

public List<String> getServerInstancesForTable(String tableName, TableType type) throws JsonParseException, JsonMappingException, JsonProcessingException, JSONException, IOException {
    String actualTableName = new TableNameBuilder(type).forTable(tableName);
    AbstractTableConfig config = null;
    if (type == TableType.REALTIME) {
        config = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), actualTableName);
    } else {
        config = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), actualTableName);
    }
    String serverTenantName = ControllerTenantNameBuilder.getTenantName(config.getTenantConfig().getServer(), type.getServerType());
    List<String> serverInstances = _helixAdmin.getInstancesInClusterWithTag(_helixClusterName, serverTenantName);
    return serverInstances;
}

Example 48 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method updateIndexingConfigFor.

public void updateIndexingConfigFor(String tableName, TableType type, IndexingConfig newConfigs) throws Exception {
    String actualTableName = new TableNameBuilder(type).forTable(tableName);
    AbstractTableConfig config;
    if (type == TableType.REALTIME) {
        config = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), actualTableName);
        if (config != null) {
            ((RealtimeTableConfig) config).setIndexConfig(newConfigs);
        }
    } else {
        config = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), actualTableName);
        if (config != null) {
            ((OfflineTableConfig) config).setIndexConfig(newConfigs);
        }
    }
    if (config == null) {
        throw new RuntimeException("tableName : " + tableName + " of type : " + type + " not found");
    }
    setTableConfig(config, actualTableName, type);
    if (type == TableType.REALTIME) {
        ensureRealtimeClusterIsSetUp(config, tableName, newConfigs);
    }
}

Example 49 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class HelixBrokerStarterTest method setUp.

@BeforeTest
public void setUp() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR);
    final String instanceId = "localhost_helixController";
    _pinotResourceManager = new PinotHelixResourceManager(ZkStarter.DEFAULT_ZK_STR, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZkStarter.DEFAULT_ZK_STR);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    Thread.sleep(3000);
    final Configuration pinotHelixBrokerProperties = DefaultHelixBrokerConfig.getDefaultBrokerConf();
    pinotHelixBrokerProperties.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, 8943);
    _helixBrokerStarter = new HelixBrokerStarter(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, pinotHelixBrokerProperties);
    Thread.sleep(1000);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 5, true);
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 1, true);
    final String tableName = "dining";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, null, null, 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    for (int i = 1; i <= 5; i++) {
        addOneSegment(tableName);
        Thread.sleep(2000);
        final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName));
        Assert.assertEquals(externalView.getPartitionSet().size(), i);
    }
}

Example 50 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotNumReplicaChanger method changeNumReplicas.

public void changeNumReplicas(final String tableName) throws Exception {
    // Get the number of replicas in the tableconfig.
    final String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    final AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(propertyStore, offlineTableName);
    final int newNumReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication());
    // Now get the idealstate, and get the number of replicas in it.
    IdealState currentIdealState = helixAdmin.getResourceIdealState(clusterName, offlineTableName);
    int currentNumReplicas = Integer.parseInt(currentIdealState.getReplicas());
    if (newNumReplicas > currentNumReplicas) {
        LOGGER.info("Increasing replicas not yet supported");
    } else if (newNumReplicas == currentNumReplicas) {
        LOGGER.info("Number of replicas ({}) match in table definition and Idealstate. Nothing to do for {}", newNumReplicas, offlineTableName);
    } else if (newNumReplicas < currentNumReplicas) {
        if (dryRun) {
            IdealState newIdealState = updateIdealState(currentIdealState, newNumReplicas);
            LOGGER.info("Final segment Assignment:");
            printSegmentAssignment(newIdealState.getRecord().getMapFields());
        } else {
            HelixHelper.updateIdealState(helixManager, offlineTableName, new Function<IdealState, IdealState>() {

                @Nullable
                @Override
                public IdealState apply(IdealState idealState) {
                    return updateIdealState(idealState, newNumReplicas);
                }
            }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
            waitForStable(offlineTableName);
            LOGGER.info("Successfully changed numReplicas to {} for table {}", newNumReplicas, offlineTableName);
            LOGGER.warn("*** You need to rebalance table {} ***", offlineTableName);
        }
    }
}