Examples with MiniZooKeeperCluster org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster used on opensource projects

Example 1 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class HBaseTestingUtility method startMiniZKCluster.

/**
   * Start a mini ZK cluster. If the property "test.hbase.zookeeper.property.clientPort" is set
   *  the port mentionned is used as the default port for ZooKeeper.
   */
private MiniZooKeeperCluster startMiniZKCluster(final File dir, final int zooKeeperServerNum, final int[] clientPortList) throws Exception {
    if (this.zkCluster != null) {
        throw new IOException("Cluster already running at " + dir);
    }
    this.passedZkCluster = false;
    this.zkCluster = new MiniZooKeeperCluster(this.getConfiguration());
    final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
    if (defPort > 0) {
        // If there is a port in the config file, we use it.
        this.zkCluster.setDefaultClientPort(defPort);
    }
    if (clientPortList != null) {
        // Ignore extra client ports
        int clientPortListSize = (clientPortList.length <= zooKeeperServerNum) ? clientPortList.length : zooKeeperServerNum;
        for (int i = 0; i < clientPortListSize; i++) {
            this.zkCluster.addClientPort(clientPortList[i]);
        }
    }
    int clientPort = this.zkCluster.startup(dir, zooKeeperServerNum);
    this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
    return this.zkCluster;
}

Example 2 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hive by apache.

the class MiniLlapCluster method serviceInit.

@Override
public void serviceInit(Configuration conf) throws IOException, InterruptedException {
    int rpcPort = 0;
    int mngPort = 0;
    int shufflePort = 0;
    int webPort = 0;
    int outputFormatServicePort = 0;
    boolean usePortsFromConf = conf.getBoolean("minillap.usePortsFromConf", false);
    LOG.info("MiniLlap configured to use ports from conf: {}", usePortsFromConf);
    if (usePortsFromConf) {
        rpcPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT);
        mngPort = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT);
        shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT);
        webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
        outputFormatServicePort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
    }
    HiveConf.setIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT, outputFormatServicePort);
    if (ownZkCluster) {
        miniZooKeeperCluster = new MiniZooKeeperCluster();
        miniZooKeeperCluster.startup(zkWorkDir);
    } else {
    // Already setup in the create method
    }
    conf.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    conf.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    conf.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    // Also add ZK settings to clusterSpecificConf to make sure these get picked up by whoever started this.
    clusterSpecificConfiguration.set(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + clusterNameTrimmed);
    clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname, miniZooKeeperCluster.getClientPort());
    LOG.info("Initializing {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
    for (int i = 0; i < numInstances; i++) {
        llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService, execBytesPerService, llapIoEnabled, ioIsDirect, ioBytesPerService, localDirs, rpcPort, mngPort, shufflePort, webPort, clusterNameTrimmed);
        llapDaemons[i].init(new Configuration(conf));
    }
    LOG.info("Initialized {} llap instances for MiniLlapCluster with name={}", numInstances, clusterNameTrimmed);
}

Example 3 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestGlobalThrottler method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1 = HBaseConfiguration.create();
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    conf1.setLong("replication.source.sleepforretries", 100);
    // Each WAL is about 120 bytes
    conf1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, 200);
    conf1.setLong("replication.source.per.peer.node.bandwidth", 100L);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    new ZooKeeperWatcher(conf1, "cluster1", null, true);
    conf2 = new Configuration(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf2, "cluster2", null, true);
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(utility2.getClusterKey());
    utility1.startMiniCluster(1, 1);
    utility2.startMiniCluster(1, 1);
    admin1.addPeer("peer1", rpc, null);
    admin1.addPeer("peer2", rpc, null);
    admin1.addPeer("peer3", rpc, null);
}

Example 4 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestHBaseTestingUtility method testMiniZooKeeperWithMultipleServers.

@Test
public void testMiniZooKeeperWithMultipleServers() throws Exception {
    HBaseTestingUtility hbt = new HBaseTestingUtility();
    // set up zookeeper cluster with 5 zk servers
    MiniZooKeeperCluster cluster2 = hbt.startMiniZKCluster(5);
    int defaultClientPort = 21818;
    cluster2.setDefaultClientPort(defaultClientPort);
    try {
        assertEquals(4, cluster2.getBackupZooKeeperServerNum());
        // killing the current active zk server
        int currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort >= defaultClientPort);
        // Check if the client port is returning a proper value
        assertTrue(cluster2.getClientPort() == currentActivePort);
        // kill another active zk server
        currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort >= defaultClientPort);
        assertTrue(cluster2.getClientPort() == currentActivePort);
        assertEquals(2, cluster2.getBackupZooKeeperServerNum());
        assertEquals(3, cluster2.getZooKeeperServerNum());
        // killing the backup zk servers
        cluster2.killOneBackupZooKeeperServer();
        cluster2.killOneBackupZooKeeperServer();
        assertEquals(0, cluster2.getBackupZooKeeperServerNum());
        assertEquals(1, cluster2.getZooKeeperServerNum());
        // killing the last zk server
        currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort == -1);
        assertTrue(cluster2.getClientPort() == currentActivePort);
        // this should do nothing.
        cluster2.killOneBackupZooKeeperServer();
        assertEquals(-1, cluster2.getBackupZooKeeperServerNum());
        assertEquals(0, cluster2.getZooKeeperServerNum());
    } finally {
        hbt.shutdownMiniZKCluster();
    }
}

Example 5 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestHBaseTestingUtility method testMiniZooKeeperWithOneServer.

@Test
public void testMiniZooKeeperWithOneServer() throws Exception {
    HBaseTestingUtility hbt = new HBaseTestingUtility();
    MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster();
    try {
        assertEquals(0, cluster1.getBackupZooKeeperServerNum());
        assertTrue((cluster1.killCurrentActiveZooKeeperServer() == -1));
    } finally {
        hbt.shutdownMiniZKCluster();
    }
}