Examples with MiniZooKeeperCluster org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster used on opensource projects

Example 11 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class HMasterCommandLine method startMaster.

private int startMaster() {
    Configuration conf = getConf();
    try {
        // and regionserver both in the one JVM.
        if (LocalHBaseCluster.isLocal(conf)) {
            DefaultMetricsSystem.setMiniClusterMode(true);
            final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf);
            File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR));
            // find out the default client port
            int zkClientPort = 0;
            // If the zookeeper client port is specified in server quorum, use it.
            String zkserver = conf.get(HConstants.ZOOKEEPER_QUORUM);
            if (zkserver != null) {
                String[] zkservers = zkserver.split(",");
                if (zkservers.length > 1) {
                    // In local mode deployment, we have the master + a region server and zookeeper server
                    // started in the same process. Therefore, we only support one zookeeper server.
                    String errorMsg = "Could not start ZK with " + zkservers.length + " ZK servers in local mode deployment. Aborting as clients (e.g. shell) will not " + "be able to find this ZK quorum.";
                    System.err.println(errorMsg);
                    throw new IOException(errorMsg);
                }
                String[] parts = zkservers[0].split(":");
                if (parts.length == 2) {
                    // the second part is the client port
                    zkClientPort = Integer.parseInt(parts[1]);
                }
            }
            // If the client port could not be find in server quorum conf, try another conf
            if (zkClientPort == 0) {
                zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
                // The client port has to be set by now; if not, throw exception.
                if (zkClientPort == 0) {
                    throw new IOException("No config value for " + HConstants.ZOOKEEPER_CLIENT_PORT);
                }
            }
            zooKeeperCluster.setDefaultClientPort(zkClientPort);
            // set the ZK tick time if specified
            int zkTickTime = conf.getInt(HConstants.ZOOKEEPER_TICK_TIME, 0);
            if (zkTickTime > 0) {
                zooKeeperCluster.setTickTime(zkTickTime);
            }
            // login the zookeeper server principal (if using security)
            ZKUtil.loginServer(conf, HConstants.ZK_SERVER_KEYTAB_FILE, HConstants.ZK_SERVER_KERBEROS_PRINCIPAL, null);
            int localZKClusterSessionTimeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10 * 1000);
            conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout);
            LOG.info("Starting a zookeeper cluster");
            int clientPort = zooKeeperCluster.startup(zkDataPath);
            if (clientPort != zkClientPort) {
                String errorMsg = "Could not start ZK at requested port of " + zkClientPort + ".  ZK was started at port: " + clientPort + ".  Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum.";
                System.err.println(errorMsg);
                throw new IOException(errorMsg);
            }
            conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
            // Need to have the zk cluster shutdown when master is shutdown.
            // Run a subclass that does the zk cluster shutdown on its way out.
            int mastersCount = conf.getInt("hbase.masters", 1);
            int regionServersCount = conf.getInt("hbase.regionservers", 1);
            // Set start timeout to 5 minutes for cmd line start operations
            conf.setIfUnset("hbase.master.start.timeout.localHBaseCluster", "300000");
            LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount + ", regionserversCount=" + regionServersCount);
            LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount, LocalHMaster.class, HRegionServer.class);
            ((LocalHMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
            cluster.startup();
            waitOnMasterThreads(cluster);
        } else {
            logProcessInfo(getConf());
            CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
            HMaster master = HMaster.constructMaster(masterClass, conf, csm);
            if (master.isStopped()) {
                LOG.info("Won't bring the Master up as a shutdown is requested");
                return 1;
            }
            master.start();
            master.join();
            if (master.isAborted())
                throw new RuntimeException("HMaster Aborted");
        }
    } catch (Throwable t) {
        LOG.error("Master exiting", t);
        return 1;
    }
    return 0;
}

Example 12 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestHBaseTestingUtility method testMiniZooKeeperWithMultipleClientPorts.

@Test
public void testMiniZooKeeperWithMultipleClientPorts() throws Exception {
    int defaultClientPort = 8888;
    int i, j;
    HBaseTestingUtility hbt = new HBaseTestingUtility();
    // Test 1 - set up zookeeper cluster with same number of ZK servers and specified client ports
    int[] clientPortList1 = { 1111, 1112, 1113 };
    MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster(clientPortList1.length, clientPortList1);
    try {
        List<Integer> clientPortListInCluster = cluster1.getClientPortList();
        for (i = 0; i < clientPortListInCluster.size(); i++) {
            assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList1[i]);
        }
    } finally {
        hbt.shutdownMiniZKCluster();
    }
    // Test 2 - set up zookeeper cluster with more ZK servers than specified client ports
    hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
    int[] clientPortList2 = { 2222, 2223 };
    MiniZooKeeperCluster cluster2 = hbt.startMiniZKCluster(clientPortList2.length + 2, clientPortList2);
    try {
        List<Integer> clientPortListInCluster = cluster2.getClientPortList();
        for (i = 0, j = 0; i < clientPortListInCluster.size(); i++) {
            if (i < clientPortList2.length) {
                assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList2[i]);
            } else {
                // servers with no specified client port will use defaultClientPort or some other ports
                // based on defaultClientPort
                assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
                j++;
            }
        }
    } finally {
        hbt.shutdownMiniZKCluster();
    }
    // Test 3 - set up zookeeper cluster with invalid client ports
    hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
    int[] clientPortList3 = { 3333, -3334, 3335, 0 };
    MiniZooKeeperCluster cluster3 = hbt.startMiniZKCluster(clientPortList3.length + 1, clientPortList3);
    try {
        List<Integer> clientPortListInCluster = cluster3.getClientPortList();
        for (i = 0, j = 0; i < clientPortListInCluster.size(); i++) {
            // the default port or a port based on default port will be used.
            if (i < clientPortList3.length && clientPortList3[i] > 0) {
                assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList3[i]);
            } else {
                assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
                j++;
            }
        }
    } finally {
        hbt.shutdownMiniZKCluster();
    }
    // Test 4 - set up zookeeper cluster with default port and some other ports used
    // This test tests that the defaultClientPort and defaultClientPort+2 are used, so
    // the algorithm should choice defaultClientPort+1 and defaultClientPort+3 to fill
    // out the ports for servers without ports specified.
    hbt.getConfiguration().setInt("test.hbase.zookeeper.property.clientPort", defaultClientPort);
    int[] clientPortList4 = { -4444, defaultClientPort + 2, 4446, defaultClientPort };
    MiniZooKeeperCluster cluster4 = hbt.startMiniZKCluster(clientPortList4.length + 1, clientPortList4);
    try {
        List<Integer> clientPortListInCluster = cluster4.getClientPortList();
        for (i = 0, j = 1; i < clientPortListInCluster.size(); i++) {
            // the default port or a port based on default port will be used.
            if (i < clientPortList4.length && clientPortList4[i] > 0) {
                assertEquals(clientPortListInCluster.get(i).intValue(), clientPortList4[i]);
            } else {
                assertEquals(clientPortListInCluster.get(i).intValue(), defaultClientPort + j);
                j += 2;
            }
        }
    } finally {
        hbt.shutdownMiniZKCluster();
    }
    // Test 5 - set up zookeeper cluster with same ports specified - fail is expected.
    int[] clientPortList5 = { 5555, 5556, 5556 };
    try {
        MiniZooKeeperCluster cluster5 = hbt.startMiniZKCluster(clientPortList5.length, clientPortList5);
        // expected failure
        assertTrue(cluster5.getClientPort() == -1);
    } catch (Exception e) {
    // exception is acceptable
    } finally {
        hbt.shutdownMiniZKCluster();
    }
}

Example 13 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestVisibilityLabelsReplication method setup.

@Before
public void setup() throws Exception {
    // setup configuration
    conf = HBaseConfiguration.create();
    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
    conf.setInt("hfile.format.version", 3);
    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    conf.setInt("replication.source.size.capacity", 10240);
    conf.setLong("replication.source.sleepforretries", 100);
    conf.setInt("hbase.regionserver.maxlogs", 10);
    conf.setLong("hbase.master.logcleaner.ttl", 10);
    conf.setInt("zookeeper.recovery.retry", 1);
    conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
    conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf.setInt("replication.stats.thread.period.seconds", 5);
    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    setVisibilityLabelServiceImpl(conf);
    conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
    VisibilityTestUtil.enableVisiblityLabels(conf);
    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, VisibilityReplication.class.getName());
    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, SimpleCP.class.getName());
    // Have to reset conf1 in case zk cluster location different
    // than default
    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
    conf.set("hbase.superuser", User.getCurrent().getShortName());
    SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(), new String[] { "supergroup" });
    // User.createUserForTesting(conf, User.getCurrent().getShortName(), new
    // String[] { "supergroup" });
    USER1 = User.createUserForTesting(conf, "user1", new String[] {});
    TEST_UTIL = new HBaseTestingUtility(conf);
    TEST_UTIL.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
    zkw1 = new ZooKeeperWatcher(conf, "cluster1", null, true);
    admin = TEST_UTIL.getAdmin();
    // Base conf2 on conf1 so it gets the right zk cluster.
    conf1 = HBaseConfiguration.create(conf);
    conf1.setInt("hfile.format.version", 3);
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
    conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSink.class.getName());
    // setVisibilityLabelServiceImpl(conf1);
    USER1 = User.createUserForTesting(conf1, "user1", new String[] {});
    TEST_UTIL1 = new HBaseTestingUtility(conf1);
    TEST_UTIL1.setZkCluster(miniZK);
    zkw2 = new ZooKeeperWatcher(conf1, "cluster2", null, true);
    TEST_UTIL.startMiniCluster(1);
    // Wait for the labels table to become available
    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
    TEST_UTIL1.startMiniCluster(1);
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(TEST_UTIL1.getClusterKey());
    admin.addReplicationPeer("2", rpc);
    Admin hBaseAdmin = TEST_UTIL.getAdmin();
    HTableDescriptor table = new HTableDescriptor(TABLE_NAME);
    HColumnDescriptor desc = new HColumnDescriptor(fam);
    desc.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    table.addFamily(desc);
    try {
        hBaseAdmin.createTable(table);
    } finally {
        if (hBaseAdmin != null) {
            hBaseAdmin.close();
        }
    }
    Admin hBaseAdmin1 = TEST_UTIL1.getAdmin();
    try {
        hBaseAdmin1.createTable(table);
    } finally {
        if (hBaseAdmin1 != null) {
            hBaseAdmin1.close();
        }
    }
    addLabels();
    setAuths(conf);
    setAuths(conf1);
}

Example 14 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestSerialReplication method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1 = HBaseConfiguration.create();
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    // smaller block size and capacity to trigger more operations
    // and test them
    conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
    conf1.setInt("replication.source.size.capacity", 1024);
    conf1.setLong("replication.source.sleepforretries", 100);
    conf1.setInt("hbase.regionserver.maxlogs", 10);
    conf1.setLong("hbase.master.logcleaner.ttl", 10);
    conf1.setBoolean("dfs.support.append", true);
    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
    // Each WAL is 120 bytes
    conf1.setLong("replication.source.per.peer.node.bandwidth", 100L);
    conf1.setLong("replication.source.size.capacity", 1L);
    conf1.setLong(HConstants.REPLICATION_SERIALLY_WAITING_KEY, 1000L);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    new ZooKeeperWatcher(conf1, "cluster1", null, true);
    conf2 = new Configuration(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf2, "cluster2", null, true);
    utility1.startMiniCluster(1, 10);
    utility2.startMiniCluster(1, 1);
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(utility2.getClusterKey());
    admin1.addPeer("1", rpc, null);
    utility1.getAdmin().setBalancerRunning(false, true);
}

Example 15 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestPerTableCFReplication method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1 = HBaseConfiguration.create();
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    // smaller block size and capacity to trigger more operations
    // and test them
    conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
    conf1.setInt("replication.source.size.capacity", 1024);
    conf1.setLong("replication.source.sleepforretries", 100);
    conf1.setInt("hbase.regionserver.maxlogs", 10);
    conf1.setLong("hbase.master.logcleaner.ttl", 10);
    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    new ZooKeeperWatcher(conf1, "cluster1", null, true);
    conf2 = new Configuration(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf3 = new Configuration(conf1);
    conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf2, "cluster3", null, true);
    utility3 = new HBaseTestingUtility(conf3);
    utility3.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf3, "cluster3", null, true);
    table = new HTableDescriptor(tableName);
    HColumnDescriptor fam = new HColumnDescriptor(famName);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    table.addFamily(fam);
    fam = new HColumnDescriptor(noRepfamName);
    table.addFamily(fam);
    tabA = new HTableDescriptor(tabAName);
    fam = new HColumnDescriptor(f1Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabA.addFamily(fam);
    fam = new HColumnDescriptor(f2Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabA.addFamily(fam);
    fam = new HColumnDescriptor(f3Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabA.addFamily(fam);
    tabB = new HTableDescriptor(tabBName);
    fam = new HColumnDescriptor(f1Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabB.addFamily(fam);
    fam = new HColumnDescriptor(f2Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabB.addFamily(fam);
    fam = new HColumnDescriptor(f3Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabB.addFamily(fam);
    tabC = new HTableDescriptor(tabCName);
    fam = new HColumnDescriptor(f1Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabC.addFamily(fam);
    fam = new HColumnDescriptor(f2Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabC.addFamily(fam);
    fam = new HColumnDescriptor(f3Name);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    tabC.addFamily(fam);
    utility1.startMiniCluster();
    utility2.startMiniCluster();
    utility3.startMiniCluster();
}