Examples with ZooKeeperWatcher org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher used on opensource projects

Example 91 with ZooKeeperWatcher

use of org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher in project hbase by apache.

the class TestZKProcedure method testMultiCohortWithMemberTimeoutDuringPrepare.

/**
   * Test a distributed commit with multiple cohort members, where one of the cohort members has a
   * timeout exception during the prepare stage.
   */
@Test
public void testMultiCohortWithMemberTimeoutDuringPrepare() throws Exception {
    String opDescription = "error injection coordination";
    String[] cohortMembers = new String[] { "one", "two", "three" };
    List<String> expected = Lists.newArrayList(cohortMembers);
    // error constants
    final int memberErrorIndex = 2;
    final CountDownLatch coordinatorReceivedErrorLatch = new CountDownLatch(1);
    // start running the coordinator and its controller
    ZooKeeperWatcher coordinatorWatcher = newZooKeeperWatcher();
    ZKProcedureCoordinator coordinatorController = new ZKProcedureCoordinator(coordinatorWatcher, opDescription, COORDINATOR_NODE_NAME);
    ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(COORDINATOR_NODE_NAME, POOL_SIZE, KEEP_ALIVE);
    ProcedureCoordinator coordinator = spy(new ProcedureCoordinator(coordinatorController, pool));
    // start a member for each node
    SubprocedureFactory subprocFactory = Mockito.mock(SubprocedureFactory.class);
    List<Pair<ProcedureMember, ZKProcedureMemberRpcs>> members = new ArrayList<>(expected.size());
    for (String member : expected) {
        ZooKeeperWatcher watcher = newZooKeeperWatcher();
        ZKProcedureMemberRpcs controller = new ZKProcedureMemberRpcs(watcher, opDescription);
        ThreadPoolExecutor pool2 = ProcedureMember.defaultPool(member, 1, KEEP_ALIVE);
        ProcedureMember mem = new ProcedureMember(controller, pool2, subprocFactory);
        members.add(new Pair<>(mem, controller));
        controller.start(member, mem);
    }
    // setup mock subprocedures
    final List<Subprocedure> cohortTasks = new ArrayList<>();
    final int[] elem = new int[1];
    for (int i = 0; i < members.size(); i++) {
        ForeignExceptionDispatcher cohortMonitor = new ForeignExceptionDispatcher();
        final ProcedureMember comms = members.get(i).getFirst();
        Subprocedure commit = Mockito.spy(new SubprocedureImpl(comms, opName, cohortMonitor, WAKE_FREQUENCY, TIMEOUT));
        // This nasty bit has one of the impls throw a TimeoutException
        Mockito.doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                int index = elem[0];
                if (index == memberErrorIndex) {
                    LOG.debug("Sending error to coordinator");
                    ForeignException remoteCause = new ForeignException("TIMER", new TimeoutException("subprocTimeout", 1, 2, 0));
                    Subprocedure r = ((Subprocedure) invocation.getMock());
                    LOG.error("Remote commit failure, not propagating error:" + remoteCause);
                    comms.receiveAbortProcedure(r.getName(), remoteCause);
                    assertEquals(r.isComplete(), true);
                    // notification (which ensures that we never progress past prepare)
                    try {
                        Procedure.waitForLatch(coordinatorReceivedErrorLatch, new ForeignExceptionDispatcher(), WAKE_FREQUENCY, "coordinator received error");
                    } catch (InterruptedException e) {
                        LOG.debug("Wait for latch interrupted, done:" + (coordinatorReceivedErrorLatch.getCount() == 0));
                        // reset the interrupt status on the thread
                        Thread.currentThread().interrupt();
                    }
                }
                elem[0] = ++index;
                return null;
            }
        }).when(commit).acquireBarrier();
        cohortTasks.add(commit);
    }
    // pass out a task per member
    final AtomicInteger taskIndex = new AtomicInteger();
    Mockito.when(subprocFactory.buildSubprocedure(Mockito.eq(opName), (byte[]) Mockito.argThat(new ArrayEquals(data)))).thenAnswer(new Answer<Subprocedure>() {

        @Override
        public Subprocedure answer(InvocationOnMock invocation) throws Throwable {
            int index = taskIndex.getAndIncrement();
            Subprocedure commit = cohortTasks.get(index);
            return commit;
        }
    });
    // setup spying on the coordinator
    ForeignExceptionDispatcher coordinatorTaskErrorMonitor = Mockito.spy(new ForeignExceptionDispatcher());
    Procedure coordinatorTask = Mockito.spy(new Procedure(coordinator, coordinatorTaskErrorMonitor, WAKE_FREQUENCY, TIMEOUT, opName, data, expected));
    when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(opName), eq(data), anyListOf(String.class))).thenReturn(coordinatorTask);
    // count down the error latch when we get the remote error
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // pass on the error to the master
            invocation.callRealMethod();
            // then count down the got error latch
            coordinatorReceivedErrorLatch.countDown();
            return null;
        }
    }).when(coordinatorTask).receive(Mockito.any(ForeignException.class));
    // ----------------------------
    // start running the operation
    // ----------------------------
    Procedure task = coordinator.startProcedure(coordinatorTaskErrorMonitor, opName, data, expected);
    assertEquals("Didn't mock coordinator task", coordinatorTask, task);
    // wait for the task to complete
    try {
        task.waitForCompleted();
    } catch (ForeignException fe) {
    // this may get caught or may not
    }
    // -------------
    // verification
    // -------------
    // always expect prepared, never committed, and possible to have cleanup and finish (racy since
    // error case)
    waitAndVerifyProc(coordinatorTask, once, never(), once, atMost(1), true);
    verifyCohortSuccessful(expected, subprocFactory, cohortTasks, once, never(), once, once, true);
    // close all the open things
    closeAll(coordinator, coordinatorController, members);
}

Example 92 with ZooKeeperWatcher

use of org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher in project hbase by apache.

the class TestZKProcedureControllers method runEarlyPrepareNodes.

// TODO Broken by composition.
//  @Test
//  public void testCoordinatorControllerHandlesEarlyPrepareNodes() throws Exception {
//    runEarlyPrepareNodes(startCoordinatorFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
//      "cohort1", "cohort2");
//    runEarlyPrepareNodes(startCohortFirst, "testEarlyPreparenodes", new byte[] { 1, 2, 3 },
//      "cohort1", "cohort2");
//  }
public void runEarlyPrepareNodes(StartControllers controllers, String operationName, byte[] data, String... cohort) throws Exception {
    ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
    List<String> expected = Lists.newArrayList(cohort);
    final Subprocedure sub = Mockito.mock(Subprocedure.class);
    Mockito.when(sub.getName()).thenReturn(operationName);
    final CountDownLatch prepared = new CountDownLatch(expected.size());
    final CountDownLatch committed = new CountDownLatch(expected.size());
    ArrayList<byte[]> dataFromMembers = new ArrayList<>();
    // mock out coordinator so we can keep track of zk progress
    ProcedureCoordinator coordinator = setupMockCoordinator(operationName, prepared, committed, dataFromMembers);
    ProcedureMember member = Mockito.mock(ProcedureMember.class);
    Procedure p = Mockito.mock(Procedure.class);
    Mockito.when(p.getName()).thenReturn(operationName);
    Pair<ZKProcedureCoordinator, List<ZKProcedureMemberRpcs>> pair = controllers.start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
    ZKProcedureCoordinator controller = pair.getFirst();
    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
    // post 1/2 the prepare nodes early
    for (int i = 0; i < cohortControllers.size() / 2; i++) {
        cohortControllers.get(i).sendMemberAcquired(sub);
    }
    // start the operation
    controller.sendGlobalBarrierAcquire(p, data, expected);
    // post the prepare node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberAcquired(sub);
    }
    // wait for all the notifications to reach the coordinator
    prepared.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString());
    // kick off the commit phase
    controller.sendGlobalBarrierReached(p, expected);
    // post the committed node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberCompleted(sub, memberData);
    }
    // wait for all commit notifications to reach the coordiantor
    committed.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString(), Mockito.eq(memberData));
    controller.resetMembers(p);
    // verify all behavior
    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
    verifyCohort(member, cohortControllers.size(), operationName, data);
    verifyCoordinator(operationName, coordinator, expected);
}

Example 93 with ZooKeeperWatcher

use of org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher in project hbase by apache.

the class TestZKProcedureControllers method runMockCommitWithOrchestratedControllers.

private void runMockCommitWithOrchestratedControllers(StartControllers controllers, String operationName, byte[] data, String... cohort) throws Exception {
    ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
    List<String> expected = Lists.newArrayList(cohort);
    final Subprocedure sub = Mockito.mock(Subprocedure.class);
    Mockito.when(sub.getName()).thenReturn(operationName);
    CountDownLatch prepared = new CountDownLatch(expected.size());
    CountDownLatch committed = new CountDownLatch(expected.size());
    ArrayList<byte[]> dataFromMembers = new ArrayList<>();
    // mock out coordinator so we can keep track of zk progress
    ProcedureCoordinator coordinator = setupMockCoordinator(operationName, prepared, committed, dataFromMembers);
    ProcedureMember member = Mockito.mock(ProcedureMember.class);
    Pair<ZKProcedureCoordinator, List<ZKProcedureMemberRpcs>> pair = controllers.start(watcher, operationName, coordinator, CONTROLLER_NODE_NAME, member, expected);
    ZKProcedureCoordinator controller = pair.getFirst();
    List<ZKProcedureMemberRpcs> cohortControllers = pair.getSecond();
    // start the operation
    Procedure p = Mockito.mock(Procedure.class);
    Mockito.when(p.getName()).thenReturn(operationName);
    controller.sendGlobalBarrierAcquire(p, data, expected);
    // post the prepare node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberAcquired(sub);
    }
    // wait for all the notifications to reach the coordinator
    prepared.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberAcquiredBarrier(Mockito.eq(operationName), Mockito.anyString());
    // kick off the commit phase
    controller.sendGlobalBarrierReached(p, expected);
    // post the committed node for each expected node
    for (ZKProcedureMemberRpcs cc : cohortControllers) {
        cc.sendMemberCompleted(sub, memberData);
    }
    // wait for all commit notifications to reach the coordinator
    committed.await();
    // make sure we got the all the nodes and no more
    Mockito.verify(coordinator, times(expected.size())).memberFinishedBarrier(Mockito.eq(operationName), Mockito.anyString(), Mockito.eq(memberData));
    assertEquals("Incorrect number of members returnd data", expected.size(), dataFromMembers.size());
    for (byte[] result : dataFromMembers) {
        assertArrayEquals("Incorrect data from member", memberData, result);
    }
    controller.resetMembers(p);
    // verify all behavior
    verifyZooKeeperClean(operationName, watcher, controller.getZkProcedureUtil());
    verifyCohort(member, cohortControllers.size(), operationName, data);
    verifyCoordinator(operationName, coordinator, expected);
}

Example 94 with ZooKeeperWatcher

use of org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher in project hbase by apache.

the class SimpleRSProcedureManager method initialize.

@Override
public void initialize(RegionServerServices rss) throws KeeperException {
    this.rss = rss;
    ZooKeeperWatcher zkw = rss.getZooKeeper();
    this.memberRpcs = new ZKProcedureMemberRpcs(zkw, getProcedureSignature());
    ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), 1);
    this.member = new ProcedureMember(memberRpcs, pool, new SimleSubprocedureBuilder());
    LOG.info("Initialized: " + rss.getServerName().toString());
}

Example 95 with ZooKeeperWatcher

use of org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher in project hbase by apache.

the class TestCompactionInDeadRegionServer method test.

@Test
public void test() throws Exception {
    HRegionServer rsToSuspend = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
    HRegion region = (HRegion) rsToSuspend.getOnlineRegions(TABLE_NAME).get(0);
    ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
    watcher.getRecoverableZooKeeper().delete(ZKUtil.joinZNode(watcher.getZNodePaths().rsZNode, rsToSuspend.getServerName().toString()), -1);
    UTIL.waitFor(60000, 1000, new ExplainingPredicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            for (RegionServerThread thread : UTIL.getHBaseCluster().getRegionServerThreads()) {
                HRegionServer rs = thread.getRegionServer();
                if (rs != rsToSuspend) {
                    return !rs.getOnlineRegions(TABLE_NAME).isEmpty();
                }
            }
            return false;
        }

        @Override
        public String explainFailure() throws Exception {
            return "The region for " + TABLE_NAME + " is still on " + rsToSuspend.getServerName();
        }
    });
    try {
        region.compact(true);
        fail("Should fail as our wal file has already been closed, " + "and walDir has also been renamed");
    } catch (Exception e) {
    // expected
    }
    Table table = UTIL.getConnection().getTable(TABLE_NAME);
    // should not hit FNFE
    for (int i = 0; i < 20; i++) {
        assertEquals(i, Bytes.toInt(table.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ)));
    }
}