Examples with PinotHelixResourceManager com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager used on opensource projects

Example 1 with PinotHelixResourceManager

use of com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager in project pinot by linkedin.

the class PerfBenchmarkDriver method startHelixResourceManager.

private void startHelixResourceManager() throws Exception {
    _helixResourceManager = new PinotHelixResourceManager(getControllerConf());
    _helixResourceManager.start();
    // Create broker tenant.
    Tenant brokerTenant = new TenantBuilder(_brokerTenantName).setRole(TenantRole.BROKER).setTotalInstances(1).build();
    _helixResourceManager.createBrokerTenant(brokerTenant);
    // Create server tenant.
    Tenant serverTenant = new TenantBuilder(_serverTenantName).setRole(TenantRole.SERVER).setTotalInstances(1).setOfflineInstances(1).build();
    _helixResourceManager.createServerTenant(serverTenant);
}

Example 2 with PinotHelixResourceManager

use of com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager in project pinot by linkedin.

the class TableViewsTest method setupTest.

@BeforeClass
public void setupTest() throws Exception {
    startZk();
    startController();
    pinotHelixResourceManager = new PinotHelixResourceManager(ZkStarter.DEFAULT_ZK_STR, getHelixClusterName(), TableViewsTest.class.getName() + "_controller", null, 10000L, true, /*isUpdateStateModel=*/
    false);
    pinotHelixResourceManager.start();
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, 5, true);
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, 20, true);
    JSONObject request = ControllerRequestBuilderUtil.buildBrokerTenantCreateRequestJSON("default", 5);
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forBrokerTenantCreate(), request.toString());
    request = ControllerRequestBuilderUtil.buildServerTenantCreateRequestJSON("default", 20, 16, 2);
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forBrokerTenantCreate(), request.toString());
    request = ControllerRequestBuilder.buildCreateOfflineTableJSON(OFFLINE_ONLY_TABLE, "default", "default", 2, "BalanceNumSegmentAssignmentStrategy");
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forTableCreate(), request.toString());
    addOneSegment(OFFLINE_ONLY_TABLE);
    request = ControllerRequestBuilder.buildCreateOfflineTableJSON(TABLE_NAME, "default", "default", 2, "BalanceNumSegmentAssignmentStrategy");
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forTableCreate(), request.toString());
    JSONObject metadata = new JSONObject();
    metadata.put("streamType", "kafka");
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.CONSUMER_TYPE, DataSource.Realtime.Kafka.ConsumerType.highLevel.toString());
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.TOPIC_NAME, "fakeTopic");
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.DECODER_CLASS, "fakeClass");
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.ZK_BROKER_URL, "fakeUrl");
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.HighLevelConsumer.ZK_CONNECTION_STRING, "potato");
    metadata.put(DataSource.Realtime.REALTIME_SEGMENT_FLUSH_SIZE, Integer.toString(1234));
    metadata.put(DataSource.STREAM_PREFIX + "." + DataSource.Realtime.Kafka.KAFKA_CONSUMER_PROPS_PREFIX + "." + DataSource.Realtime.Kafka.AUTO_OFFSET_RESET, "smallest");
    request = ControllerRequestBuilder.buildCreateRealtimeTableJSON(TABLE_NAME, "default", "default", "potato", "DAYS", "DAYS", "5", 2, "BalanceNumSegmentAssignmentStrategy", metadata, "fakeSchema", "fakeColumn", Collections.<String>emptyList(), "MMAP", true);
    sendPostRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forTableCreate(), request.toString());
}

Example 3 with PinotHelixResourceManager

use of com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager in project pinot by linkedin.

the class HelixBrokerStarterTest method setUp.

@BeforeTest
public void setUp() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR);
    final String instanceId = "localhost_helixController";
    _pinotResourceManager = new PinotHelixResourceManager(ZkStarter.DEFAULT_ZK_STR, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZkStarter.DEFAULT_ZK_STR);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    Thread.sleep(3000);
    final Configuration pinotHelixBrokerProperties = DefaultHelixBrokerConfig.getDefaultBrokerConf();
    pinotHelixBrokerProperties.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, 8943);
    _helixBrokerStarter = new HelixBrokerStarter(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, pinotHelixBrokerProperties);
    Thread.sleep(1000);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 5, true);
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 1, true);
    final String tableName = "dining";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, null, null, 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    for (int i = 1; i <= 5; i++) {
        addOneSegment(tableName);
        Thread.sleep(2000);
        final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName));
        Assert.assertEquals(externalView.getPartitionSet().size(), i);
    }
}

Example 4 with PinotHelixResourceManager

use of com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager in project pinot by linkedin.

the class RetentionManagerTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_STR);
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    _helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    _helixZkManager = _pinotHelixResourceManager.getHelixZkManager();
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(_testTableName, null, null, 2).toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
    _propertyStore = ZkUtils.getZkPropertyStore(_helixZkManager, HELIX_CLUSTER_NAME);
}

Example 5 with PinotHelixResourceManager

use of com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager in project pinot by linkedin.

the class SegmentAssignmentStrategyTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_SERVER);
    final String zkPath = "/" + HELIX_CLUSTER_NAME;
    if (_zkClient.exists(zkPath)) {
        _zkClient.deleteRecursive(zkPath);
    }
    final String instanceId = "localhost_helixController";
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_SERVER, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZK_SERVER);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, _numServerInstance, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, _numBrokerInstance, true);
    Thread.sleep(3000);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_OFFLINE").size(), _numServerInstance);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_REALTIME").size(), _numServerInstance);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), _numBrokerInstance);
}