Examples with ZkClient org.apache.helix.manager.zk.ZkClient used on opensource projects

Example 51 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class OfflineClusterIntegrationTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    //Clean up
    ensureDirectoryExistsAndIsEmpty(_tmpDir);
    ensureDirectoryExistsAndIsEmpty(_segmentDir);
    ensureDirectoryExistsAndIsEmpty(_tarDir);
    // Start the cluster
    startCluster();
    // Unpack the Avro files
    final List<File> avroFiles = unpackAvroData(_tmpDir, SEGMENT_COUNT);
    createTable();
    // Get the list of instances through the REST API
    URL url = new URL("http://" + ControllerTestUtils.DEFAULT_CONTROLLER_HOST + ":" + ControllerTestUtils.DEFAULT_CONTROLLER_API_PORT + "/instances");
    InputStream inputStream = url.openConnection().getInputStream();
    String instanceApiResponseString = IOUtils.toString(inputStream);
    IOUtils.closeQuietly(inputStream);
    JSONObject instanceApiResponse = new JSONObject(instanceApiResponseString);
    JSONArray instanceArray = instanceApiResponse.getJSONArray("instances");
    HelixAdmin helixAdmin = new ZKHelixAdmin(new ZkClient(ZkStarter.DEFAULT_ZK_STR, 10000, 10000, new ZNRecordSerializer()));
    for (int i = 0; i < instanceArray.length(); i++) {
        String instance = instanceArray.getString(i);
        if (instance.startsWith("Server_")) {
            _serverServiceStatusCallback = new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(helixAdmin, getHelixClusterName(), instance);
        }
        if (instance.startsWith("Broker_")) {
            _brokerServiceStatusCallback = new ServiceStatus.IdealStateAndExternalViewMatchServiceStatusCallback(helixAdmin, getHelixClusterName(), instance, Collections.singletonList("brokerResource"));
        }
    }
    // Load data into H2
    ExecutorService executor = Executors.newCachedThreadPool();
    setupH2AndInsertAvro(avroFiles, executor);
    // Create segments from Avro data
    buildSegmentsFromAvro(avroFiles, executor, 0, _segmentDir, _tarDir, "mytable", false, null);
    // Initialize query generator
    setupQueryGenerator(avroFiles, executor);
    executor.shutdown();
    executor.awaitTermination(10, TimeUnit.MINUTES);
    // Set up a Helix spectator to count the number of segments that are uploaded and unlock the latch once 12 segments are online
    final CountDownLatch latch = setupSegmentCountCountDownLatch("mytable", SEGMENT_COUNT);
    // Upload the segments
    int i = 0;
    for (String segmentName : _tarDir.list()) {
        //      System.out.println("Uploading segment " + (i++) + " : " + segmentName);
        File file = new File(_tarDir, segmentName);
        FileUploadUtils.sendSegmentFile("localhost", "8998", segmentName, file, file.length());
    }
    // Wait for all segments to be online
    latch.await();
    TOTAL_DOCS = 115545;
    long timeInTwoMinutes = System.currentTimeMillis() + 2 * 60 * 1000L;
    long numDocs;
    while ((numDocs = getCurrentServingNumDocs("mytable")) < TOTAL_DOCS) {
        //      System.out.println("Current number of documents: " + numDocs);
        if (System.currentTimeMillis() < timeInTwoMinutes) {
            Thread.sleep(1000);
        } else {
            Assert.fail("Segments were not completely loaded within two minutes");
        }
    }
}

Example 52 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class RetentionManagerTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_STR);
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    _helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    _helixZkManager = _pinotHelixResourceManager.getHelixZkManager();
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(_testTableName, null, null, 2).toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
    _propertyStore = ZkUtils.getZkPropertyStore(_helixZkManager, HELIX_CLUSTER_NAME);
}

Example 53 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class ValidationManagerTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_STR);
    Thread.sleep(1000);
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 1000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true);
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TEST_TABLE_NAME, null, null, "timestamp", "millsSinceEpoch", "DAYS", "5", 2, "BalanceNumSegmentAssignmentStrategy").toString();
    _offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    final String instanceId = "localhost_helixController";
    _helixManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, ZK_STR, instanceId, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.addTable(_offlineTableConfig);
}

Example 54 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class PinotResourceManagerTest method setup.

@BeforeTest
public void setup() throws Exception {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(ZK_SERVER);
    final String instanceId = "localhost_helixController";
    _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_SERVER, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/
    false);
    _pinotHelixResourceManager.start();
    final String helixZkURL = HelixConfig.getAbsoluteZkPathForHelix(ZK_SERVER);
    _helixZkManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, helixZkURL, instanceId, /*isUpdateStateModel=*/
    false);
    _helixAdmin = _helixZkManager.getClusterManagmentTool();
    /////////////////////////
    _numInstance = 1;
    ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, _numInstance, true);
    ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_SERVER, 1, true);
    Thread.sleep(3000);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 1);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_OFFLINE").size(), 1);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_REALTIME").size(), 1);
    // Adding table
    String OfflineTableConfigJson = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TABLE_NAME, null, null, 1).toString();
    AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
    _pinotHelixResourceManager.addTable(offlineTableConfig);
}

Example 55 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class ControllerTest method startController.

/**
   * Starts a controller instance.
   */
protected void startController(ControllerConf config) {
    Preconditions.checkNotNull(config);
    Preconditions.checkState(_controllerStarter == null);
    _zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR);
    if (_zkClient.exists("/" + getHelixClusterName())) {
        _zkClient.deleteRecursive("/" + getHelixClusterName());
    }
    _controllerStarter = ControllerTestUtils.startController(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, config);
    _helixAdmin = _controllerStarter.getHelixResourceManager().getHelixAdmin();
    _propertyStore = _controllerStarter.getHelixResourceManager().getPropertyStore();
}