Examples with AbstractTableConfig com.linkedin.pinot.common.config.AbstractTableConfig used on opensource projects

Example 36 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class RetentionManagerTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_STR);
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    _helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    _helixZkManager = _pinotHelixResourceManager.getHelixZkManager();
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(_testTableName, null, null, 2).toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
    _propertyStore = ZkUtils.getZkPropertyStore(_helixZkManager, HELIX_CLUSTER_NAME);
}

Example 37 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class RetentionManagerTest method createRealtimeTableConfig.

private void createRealtimeTableConfig(String realtimeTableName, int replicaCount) throws JSONException, IOException {
    JSONObject tableConfig = new JSONObject();
    tableConfig.put("tableName", _testTableName);
    tableConfig.put("tableType", "REALTIME");
    JSONObject segmentsConfig = new JSONObject();
    segmentsConfig.put("retentionTimeUnit", "DAYS");
    segmentsConfig.put("retentionTimeValue", "5");
    segmentsConfig.put("replicasPerPartition", Integer.toString(replicaCount));
    segmentsConfig.put("replicationNumber", Integer.toString(replicaCount));
    segmentsConfig.put("replication", Integer.toString(replicaCount));
    tableConfig.put("segmentsConfig", segmentsConfig);
    JSONObject tableIndexConfig = new JSONObject();
    tableIndexConfig.put("stream.kafka.consumer.type", "simple");
    tableConfig.put("tableIndexConfig", tableIndexConfig);
    JSONObject tenants = new JSONObject();
    tableConfig.put("tenants", tenants);
    JSONObject metadata = new JSONObject();
    tableConfig.put("metadata", tenants);
    // Set the propertystore entry for table.
    AbstractTableConfig abstractTableConfig = AbstractTableConfig.init(tableConfig.toJSONString());
    ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, realtimeTableName, AbstractTableConfig.toZnRecord(abstractTableConfig));
}

Example 38 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class SegmentAssignmentStrategyTest method testRandomSegmentAssignmentStrategy.

@Test
public void testRandomSegmentAssignmentStrategy() throws Exception {
    final int numReplicas = 2;
    // Adding table
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TABLE_NAME_RANDOM, null, null, numReplicas, "RandomAssignmentStrategy").toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
    Thread.sleep(3000);
    for (int i = 0; i < 10; ++i) {
        addOneSegment(TABLE_NAME_RANDOM);
        Thread.sleep(3000);
        final Set<String> taggedInstances = _pinotHelixResourceManager.getAllInstancesForServerTenant("DefaultTenant_OFFLINE");
        final Map<String, Integer> instance2NumSegmentsMap = new HashMap<String, Integer>();
        for (final String instance : taggedInstances) {
            instance2NumSegmentsMap.put(instance, 0);
        }
        final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME_RANDOM));
        Assert.assertEquals(externalView.getPartitionSet().size(), i + 1);
        for (final String segmentId : externalView.getPartitionSet()) {
            Assert.assertEquals(externalView.getStateMap(segmentId).size(), numReplicas);
        }
    }
}

Example 39 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class SegmentAssignmentStrategyTest method testBalanceNumSegmentAssignmentStrategy.

@Test
public void testBalanceNumSegmentAssignmentStrategy() throws Exception {
    final int numReplicas = 3;
    // Adding table
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TABLE_NAME_BALANCED, null, null, numReplicas, "BalanceNumSegmentAssignmentStrategy").toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
    Thread.sleep(3000);
    int numSegments = 20;
    for (int i = 0; i < numSegments; ++i) {
        addOneSegment(TABLE_NAME_BALANCED);
        Thread.sleep(2000);
    }
    final Set<String> taggedInstances = _pinotHelixResourceManager.getAllInstancesForServerTenant("DefaultTenant_OFFLINE");
    final Map<String, Integer> instance2NumSegmentsMap = new HashMap<String, Integer>();
    for (final String instance : taggedInstances) {
        instance2NumSegmentsMap.put(instance, 0);
    }
    final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME_BALANCED));
    for (final String segmentId : externalView.getPartitionSet()) {
        for (final String instance : externalView.getStateMap(segmentId).keySet()) {
            instance2NumSegmentsMap.put(instance, instance2NumSegmentsMap.get(instance) + 1);
        }
    }
    final int totalSegments = (numSegments) * numReplicas;
    final int minNumSegmentsPerInstance = totalSegments / _numServerInstance;
    int maxNumSegmentsPerInstance = minNumSegmentsPerInstance;
    if ((minNumSegmentsPerInstance * _numServerInstance) < totalSegments) {
        maxNumSegmentsPerInstance = maxNumSegmentsPerInstance + 1;
    }
    for (final String instance : instance2NumSegmentsMap.keySet()) {
        Assert.assertTrue(instance2NumSegmentsMap.get(instance) >= minNumSegmentsPerInstance, "expected >=" + minNumSegmentsPerInstance + " actual:" + instance2NumSegmentsMap.get(instance));
        Assert.assertTrue(instance2NumSegmentsMap.get(instance) <= maxNumSegmentsPerInstance, "expected <=" + maxNumSegmentsPerInstance + " actual:" + instance2NumSegmentsMap.get(instance));
    }
    _helixAdmin.dropResource(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME_BALANCED));
}

Example 40 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotResourceManagerTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_SERVER);
    final String instanceId = "localhost_helixController";
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_SERVER, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZK_SERVER);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    /////////////////////////
    _numInstance = 1;
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, _numInstance, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, 1, true);
    Thread.sleep(3000);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 1);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_OFFLINE").size(), 1);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_REALTIME").size(), 1);
    // Adding table
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TABLE_NAME, null, null, 1).toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
}