Examples with ZKWatcher org.apache.hadoop.hbase.zookeeper.ZKWatcher used on opensource projects

Example 56 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestZKProcedureControllers method runEarlyPrepareNodes.

// TODO Broken by composition.
// @Test
// public void testCoordinatorControllerHandlesEarlyPrepareNodes() throws Exception {
// runEarlyPrepareNodes(startCoordinatorFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
// "cohort1", "cohort2");
// runEarlyPrepareNodes(startCohortFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
// "cohort1", "cohort2");
// }
public void runEarlyPrepareNodes(StartControllers controllers, String operationName, byte[] data, String... cohort) throws Exception {
    ZKWatcher watcher = UTIL.getZooKeeperWatcher();
    List<String> expected = Lists.newArrayList(cohort);
    final Subprocedure sub = Mockito.mock(Subprocedure.class);
    Mockito.when(sub.getName()).thenReturn(operationName);
    final CountDownLatch prepared = new CountDownLatch(expected.size());
    final CountDownLatch committed = new CountDownLatch(expected.size());
    ArrayList<byte[]> dataFromMembers = new ArrayList<>();
    // mock out coordinator so we can keep track of zk progress
    ProcedureCoordinator coordinator = setupMockCoordinator(operationName, prepared, committed, dataFromMembers);
    ProcedureMember member = Mockito.mock(ProcedureMember.class);
    Procedure p = Mockito.mock(Procedure.class);
    Mockito.when(p.getName()).thenReturn(operationName);
    Pair<ZKProcedureCoordinator, List<ZKProcedureMemberRpcs>> pair = controllers.start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
    ZKProcedureCoordinator controller = pair.getFirst();
    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
    // post 1/2 the prepare nodes early
    for (int i = 0; i < cohortControllers.size() / 2; i++) {
        cohortControllers.get(i).sendMemberAcquired(sub);
    }
    // start the operation
    controller.sendGlobalBarrierAcquire(p, data, expected);
    // post the prepare node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberAcquired(sub);
    }
    // wait for all the notifications to reach the coordinator
    prepared.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString());
    // kick off the commit phase
    controller.sendGlobalBarrierReached(p, expected);
    // post the committed node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberCompleted(sub, memberData);
    }
    // wait for all commit notifications to reach the coordiantor
    committed.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString(), Mockito.eq(memberData));
    controller.resetMembers(p);
    // verify all behavior
    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
    verifyCohort(member, cohortControllers.size(), operationName, data);
    verifyCoordinator(operationName, coordinator, expected);
}

Example 57 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestZKProcedureControllers method runMockCommitWithOrchestratedControllers.

private void runMockCommitWithOrchestratedControllers(StartControllers controllers, String operationName, byte[] data, String... cohort) throws Exception {
    ZKWatcher watcher = UTIL.getZooKeeperWatcher();
    List<String> expected = Lists.newArrayList(cohort);
    final Subprocedure sub = Mockito.mock(Subprocedure.class);
    Mockito.when(sub.getName()).thenReturn(operationName);
    CountDownLatch prepared = new CountDownLatch(expected.size());
    CountDownLatch committed = new CountDownLatch(expected.size());
    ArrayList<byte[]> dataFromMembers = new ArrayList<>();
    // mock out coordinator so we can keep track of zk progress
    ProcedureCoordinator coordinator = setupMockCoordinator(operationName, prepared, committed, dataFromMembers);
    ProcedureMember member = Mockito.mock(ProcedureMember.class);
    Pair<ZKProcedureCoordinator, List<ZKProcedureMemberRpcs>> pair = controllers.start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
    ZKProcedureCoordinator controller = pair.getFirst();
    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
    // start the operation
    Procedure p = Mockito.mock(Procedure.class);
    Mockito.when(p.getName()).thenReturn(operationName);
    controller.sendGlobalBarrierAcquire(p, data, expected);
    // post the prepare node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberAcquired(sub);
    }
    // wait for all the notifications to reach the coordinator
    prepared.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString());
    // kick off the commit phase
    controller.sendGlobalBarrierReached(p, expected);
    // post the committed node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberCompleted(sub, memberData);
    }
    // wait for all commit notifications to reach the coordinator
    committed.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString(), Mockito.eq(memberData));
    assertEquals("Incorrect number of members returnd data", expected.size(), dataFromMembers.size());
    for (byte[] result : dataFromMembers) {
        assertArrayEquals("Incorrect data from member", memberData, result);
    }
    controller.resetMembers(p);
    // verify all behavior
    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
    verifyCohort(member, cohortControllers.size(), operationName, data);
    verifyCoordinator(operationName, coordinator, expected);
}

Example 58 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class SimpleRSProcedureManager method initialize.

@Override
public void initialize(RegionServerServices rss) throws KeeperException {
    this.rss = rss;
    ZKWatcher zkw = rss.getZooKeeper();
    this.memberRpcs = new ZKProcedureMemberRpcs(zkw, getProcedureSignature());
    ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), 1);
    this.member = new ProcedureMember(memberRpcs, pool, new SimleSubprocedureBuilder());
    LOG.info("Initialized: " + rss.getServerName().toString());
}

Example 59 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestMasterReplication method startMiniClusters.

@SuppressWarnings("resource")
private void startMiniClusters(int numClusters) throws Exception {
    Random random = new Random();
    utilities = new HBaseTestingUtil[numClusters];
    configurations = new Configuration[numClusters];
    for (int i = 0; i < numClusters; i++) {
        Configuration conf = new Configuration(baseConfiguration);
        conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/" + i + random.nextInt());
        HBaseTestingUtil utility = new HBaseTestingUtil(conf);
        if (i == 0) {
            utility.startMiniZKCluster();
            miniZK = utility.getZkCluster();
        } else {
            utility.setZkCluster(miniZK);
        }
        utility.startMiniCluster();
        utilities[i] = utility;
        configurations[i] = conf;
        new ZKWatcher(conf, "cluster" + i, null, true);
    }
}

Example 60 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestZKSecretWatcher method testKeyUpdate.

@Test
public void testKeyUpdate() throws Exception {
    // sanity check
    assertTrue(KEY_MASTER.isMaster());
    assertFalse(KEY_SLAVE.isMaster());
    int maxKeyId = 0;
    KEY_MASTER.rollCurrentKey();
    AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
    assertNotNull(key1);
    LOG.debug("Master current key (key1) {}", key1);
    // wait for slave to update
    Thread.sleep(1000);
    AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
    assertNotNull(slaveCurrent);
    assertEquals(key1, slaveCurrent);
    LOG.debug("Slave current key (key1) {}", slaveCurrent);
    // generate two more keys then expire the original
    KEY_MASTER.rollCurrentKey();
    AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
    LOG.debug("Master new current key (key2) {}", key2);
    KEY_MASTER.rollCurrentKey();
    AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
    LOG.debug("Master new current key (key3) {}", key3);
    // force expire the original key
    key1.setExpiration(EnvironmentEdgeManager.currentTime() - 100000);
    KEY_MASTER.removeExpiredKeys();
    // verify removed from master
    assertNull(KEY_MASTER.getKey(key1.getKeyId()));
    // Wait for slave to catch up. When remove hits KEY_SLAVE, we'll clear
    // the latch and will progress beyond the await.
    KEY_SLAVE.getLatch().await();
    // make sure the slave has both new keys
    AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
    assertNotNull(slave2);
    assertEquals(key2, slave2);
    AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
    assertNotNull(slave3);
    assertEquals(key3, slave3);
    slaveCurrent = KEY_SLAVE.getCurrentKey();
    assertEquals(key3, slaveCurrent);
    LOG.debug("Slave current key (key3) {}", slaveCurrent);
    // verify that the expired key has been removed
    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, () -> {
        AuthenticationKey k = KEY_SLAVE.getKey(key1.getKeyId());
        LOG.info("AuthKey1={}", k);
        return k == null;
    });
    assertNull("key1=" + KEY_SLAVE.getKey(key1.getKeyId()), KEY_SLAVE.getKey(key1.getKeyId()));
    // bring up a new slave
    Configuration conf = TEST_UTIL.getConfiguration();
    ZKWatcher zk = newZK(conf, "server3", new MockAbortable());
    KEY_SLAVE2 = new AuthenticationTokenSecretManager(conf, zk, "server3", 60 * 60 * 1000, 60 * 1000);
    KEY_SLAVE2.start();
    Thread.sleep(1000);
    // verify the new slave has current keys (and not expired)
    slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
    assertNotNull(slave2);
    assertEquals(key2, slave2);
    slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
    assertNotNull(slave3);
    assertEquals(key3, slave3);
    slaveCurrent = KEY_SLAVE2.getCurrentKey();
    assertEquals(key3, slaveCurrent);
    assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
    // test leader failover
    KEY_MASTER.stop();
    // wait for master to stop
    Thread.sleep(1000);
    assertFalse(KEY_MASTER.isMaster());
    // check for a new master
    AuthenticationTokenSecretManager[] mgrs = new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2 };
    AuthenticationTokenSecretManager newMaster = null;
    int tries = 0;
    while (newMaster == null && tries++ < 5) {
        for (AuthenticationTokenSecretManager mgr : mgrs) {
            if (mgr.isMaster()) {
                newMaster = mgr;
                break;
            }
        }
        if (newMaster == null) {
            Thread.sleep(500);
        }
    }
    assertNotNull(newMaster);
    AuthenticationKey current = newMaster.getCurrentKey();
    // new master will immediately roll the current key, so it's current may be greater
    assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
    LOG.debug("New master, current key: " + current.getKeyId());
    // roll the current key again on new master and verify the key ID increments
    newMaster.rollCurrentKey();
    AuthenticationKey newCurrent = newMaster.getCurrentKey();
    LOG.debug("New master, rolled new current key: " + newCurrent.getKeyId());
    assertTrue(newCurrent.getKeyId() > current.getKeyId());
    // add another slave
    ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable());
    KEY_SLAVE3 = new AuthenticationTokenSecretManager(conf, zk3, "server4", 60 * 60 * 1000, 60 * 1000);
    KEY_SLAVE3.start();
    Thread.sleep(5000);
    // check master failover again
    newMaster.stop();
    // wait for master to stop
    Thread.sleep(5000);
    assertFalse(newMaster.isMaster());
    // check for a new master
    mgrs = new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
    newMaster = null;
    tries = 0;
    while (newMaster == null && tries++ < 5) {
        for (AuthenticationTokenSecretManager mgr : mgrs) {
            if (mgr.isMaster()) {
                newMaster = mgr;
                break;
            }
        }
        if (newMaster == null) {
            Thread.sleep(500);
        }
    }
    assertNotNull(newMaster);
    AuthenticationKey current2 = newMaster.getCurrentKey();
    // new master will immediately roll the current key, so it's current may be greater
    assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
    LOG.debug("New master 2, current key: " + current2.getKeyId());
    // roll the current key again on new master and verify the key ID increments
    newMaster.rollCurrentKey();
    AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
    LOG.debug("New master 2, rolled new current key: " + newCurrent2.getKeyId());
    assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
}