Examples with MiniZooKeeperCluster org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster used on opensource projects

Example 6 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestHBaseTestingUtility method testMiniZooKeeperWithMultipleServers.

@Test
public void testMiniZooKeeperWithMultipleServers() throws Exception {
    HBaseTestingUtility hbt = new HBaseTestingUtility();
    // set up zookeeper cluster with 5 zk servers
    MiniZooKeeperCluster cluster2 = hbt.startMiniZKCluster(5);
    int defaultClientPort = 21818;
    cluster2.setDefaultClientPort(defaultClientPort);
    try {
        assertEquals(4, cluster2.getBackupZooKeeperServerNum());
        // killing the current active zk server
        int currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort >= defaultClientPort);
        // Check if the client port is returning a proper value
        assertTrue(cluster2.getClientPort() == currentActivePort);
        // kill another active zk server
        currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort >= defaultClientPort);
        assertTrue(cluster2.getClientPort() == currentActivePort);
        assertEquals(2, cluster2.getBackupZooKeeperServerNum());
        assertEquals(3, cluster2.getZooKeeperServerNum());
        // killing the backup zk servers
        cluster2.killOneBackupZooKeeperServer();
        cluster2.killOneBackupZooKeeperServer();
        assertEquals(0, cluster2.getBackupZooKeeperServerNum());
        assertEquals(1, cluster2.getZooKeeperServerNum());
        // killing the last zk server
        currentActivePort = cluster2.killCurrentActiveZooKeeperServer();
        assertTrue(currentActivePort == -1);
        assertTrue(cluster2.getClientPort() == currentActivePort);
        // this should do nothing.
        cluster2.killOneBackupZooKeeperServer();
        assertEquals(-1, cluster2.getBackupZooKeeperServerNum());
        assertEquals(0, cluster2.getZooKeeperServerNum());
    } finally {
        hbt.shutdownMiniZKCluster();
    }
}

Example 7 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestHBaseTestingUtility method testMiniZooKeeperWithOneServer.

@Test
public void testMiniZooKeeperWithOneServer() throws Exception {
    HBaseTestingUtility hbt = new HBaseTestingUtility();
    MiniZooKeeperCluster cluster1 = hbt.startMiniZKCluster();
    try {
        assertEquals(0, cluster1.getBackupZooKeeperServerNum());
        assertTrue((cluster1.killCurrentActiveZooKeeperServer() == -1));
    } finally {
        hbt.shutdownMiniZKCluster();
    }
}

Example 8 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestReplicationBase method setUpBeforeClass.

/**
   * @throws java.lang.Exception
   */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    // We don't want too many edits per batch sent to the ReplicationEndpoint to trigger
    // sufficient number of events. But we don't want to go too low because
    // HBaseInterClusterReplicationEndpoint partitions entries into batches and we want
    // more than one batch sent to the peer cluster for better testing.
    conf1.setInt("replication.source.size.capacity", 102400);
    conf1.setLong("replication.source.sleepforretries", 100);
    conf1.setInt("hbase.regionserver.maxlogs", 10);
    conf1.setLong("hbase.master.logcleaner.ttl", 10);
    conf1.setInt("zookeeper.recovery.retry", 1);
    conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf1.setInt("replication.stats.thread.period.seconds", 5);
    conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    conf1.setLong("replication.sleep.before.failover", 2000);
    conf1.setInt("replication.source.maxretriesmultiplier", 10);
    conf1.setFloat("replication.source.ratio", 1.0f);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    // Have to reget conf1 in case zk cluster location different
    // than default
    conf1 = utility1.getConfiguration();
    zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
    admin = new ReplicationAdmin(conf1);
    LOG.info("Setup first Zk");
    // Base conf2 on conf1 so it gets the right zk cluster.
    conf2 = HBaseConfiguration.create(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
    conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
    LOG.info("Setup second Zk");
    CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
    utility1.startMiniCluster(2);
    // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks
    // as a component in deciding maximum number of parallel batches to send to the peer cluster.
    utility2.startMiniCluster(4);
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(utility2.getClusterKey());
    hbaseAdmin = ConnectionFactory.createConnection(conf1).getAdmin();
    hbaseAdmin.addReplicationPeer("2", rpc);
    HTableDescriptor table = new HTableDescriptor(tableName);
    HColumnDescriptor fam = new HColumnDescriptor(famName);
    fam.setMaxVersions(100);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    table.addFamily(fam);
    fam = new HColumnDescriptor(noRepfamName);
    table.addFamily(fam);
    scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (HColumnDescriptor f : table.getColumnFamilies()) {
        scopes.put(f.getName(), f.getScope());
    }
    Connection connection1 = ConnectionFactory.createConnection(conf1);
    Connection connection2 = ConnectionFactory.createConnection(conf2);
    try (Admin admin1 = connection1.getAdmin()) {
        admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
    }
    try (Admin admin2 = connection2.getAdmin()) {
        admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
    }
    utility1.waitUntilAllRegionsAssigned(tableName);
    utility2.waitUntilAllRegionsAssigned(tableName);
    htable1 = connection1.getTable(tableName);
    htable1.setWriteBufferSize(1024);
    htable2 = connection2.getTable(tableName);
}

Example 9 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project hbase by apache.

the class TestMultiSlaveReplication method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1 = HBaseConfiguration.create();
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    // smaller block size and capacity to trigger more operations
    // and test them
    conf1.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
    conf1.setInt("replication.source.size.capacity", 1024);
    conf1.setLong("replication.source.sleepforretries", 100);
    conf1.setInt("hbase.regionserver.maxlogs", 10);
    conf1.setLong("hbase.master.logcleaner.ttl", 10);
    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
    conf1.setInt("hbase.master.cleaner.interval", 5 * 1000);
    conf1.setClass("hbase.region.replica.replication.replicationQueues.class", ReplicationQueuesZKImpl.class, ReplicationQueues.class);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    utility1.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf1, "cluster1", null, true);
    conf2 = new Configuration(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf3 = new Configuration(conf1);
    conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf2, "cluster2", null, true);
    utility3 = new HBaseTestingUtility(conf3);
    utility3.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf3, "cluster3", null, true);
    table = new HTableDescriptor(tableName);
    HColumnDescriptor fam = new HColumnDescriptor(famName);
    fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
    table.addFamily(fam);
    fam = new HColumnDescriptor(noRepfamName);
    table.addFamily(fam);
}

Example 10 with MiniZooKeeperCluster

use of org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster in project phoenix by apache.

the class MutableIndexReplicationIT method setupConfigsAndStartCluster.

private static void setupConfigsAndStartCluster() throws Exception {
    // cluster-1 lives at regular HBase home, so we don't need to change how phoenix handles
    // lookups
    //        conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    // smaller log roll size to trigger more events
    setUpConfigForMiniCluster(conf1);
    conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
    conf1.setInt("replication.source.size.capacity", 10240);
    conf1.setLong("replication.source.sleepforretries", 100);
    conf1.setInt("hbase.regionserver.maxlogs", 10);
    conf1.setLong("hbase.master.logcleaner.ttl", 10);
    conf1.setInt("zookeeper.recovery.retry", 1);
    conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
    conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
    conf1.setBoolean("dfs.support.append", true);
    conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf1.setInt("replication.stats.thread.period.seconds", 5);
    conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    // Have to reset conf1 in case zk cluster location different
    // than default
    conf1 = utility1.getConfiguration();
    zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
    admin = new ReplicationAdmin(conf1);
    LOG.info("Setup first Zk");
    // Base conf2 on conf1 so it gets the right zk cluster, and general cluster configs
    conf2 = HBaseConfiguration.create(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
    conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
    conf2.setBoolean("dfs.support.append", true);
    conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
    //replicate from cluster 1 -> cluster 2, but not back again
    admin.addPeer("1", utility2.getClusterKey());
    LOG.info("Setup second Zk");
    utility1.startMiniCluster(2);
    utility2.startMiniCluster(2);
}