Examples with ZKWatcher org.apache.hadoop.hbase.zookeeper.ZKWatcher used on opensource projects

Example 16 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class ReplicationSyncUp method run.

@Override
public int run(String[] args) throws Exception {
    Abortable abortable = new Abortable() {

        @Override
        public void abort(String why, Throwable e) {
        }

        @Override
        public boolean isAborted() {
            return false;
        }
    };
    Configuration conf = getConf();
    try (ZKWatcher zkw = new ZKWatcher(conf, "syncupReplication" + EnvironmentEdgeManager.currentTime(), abortable, true)) {
        Path walRootDir = CommonFSUtils.getWALRootDir(conf);
        FileSystem fs = CommonFSUtils.getWALFileSystem(conf);
        Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
        Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
        System.out.println("Start Replication Server start");
        Replication replication = new Replication();
        replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir, new WALFactory(conf, "test", null, false));
        ReplicationSourceManager manager = replication.getReplicationManager();
        manager.init();
        claimReplicationQueues(zkw, manager);
        while (manager.activeFailoverTaskCount() > 0) {
            Thread.sleep(SLEEP_TIME);
        }
        while (manager.getOldSources().size() > 0) {
            Thread.sleep(SLEEP_TIME);
        }
        manager.join();
    } catch (InterruptedException e) {
        System.err.println("didn't wait long enough:" + e);
        return -1;
    }
    return 0;
}

Example 17 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestZKSecretWatcher method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws Exception {
    TEST_UTIL = new HBaseTestingUtil();
    TEST_UTIL.startMiniZKCluster();
    Configuration conf = TEST_UTIL.getConfiguration();
    ZKWatcher zk = newZK(conf, "server1", new MockAbortable());
    AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
    tmp[0] = new AuthenticationTokenSecretManagerForTest(conf, zk, "server1", 60 * 60 * 1000, 60 * 1000);
    tmp[0].start();
    zk = newZK(conf, "server2", new MockAbortable());
    tmp[1] = new AuthenticationTokenSecretManagerForTest(conf, zk, "server2", 60 * 60 * 1000, 60 * 1000);
    tmp[1].start();
    while (KEY_MASTER == null) {
        for (int i = 0; i < 2; i++) {
            if (tmp[i].isMaster()) {
                KEY_MASTER = tmp[i];
                KEY_SLAVE = tmp[(i + 1) % 2];
                break;
            }
        }
        Thread.sleep(500);
    }
    LOG.info("Master is " + KEY_MASTER.getName() + ", slave is " + KEY_SLAVE.getName());
}

Example 18 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestVisibilityLabelReplicationWithExpAsString method setup.

@Override
@Before
public void setup() throws Exception {
    expected[0] = 4;
    expected[1] = 6;
    expected[2] = 4;
    expected[3] = 0;
    expected[3] = 3;
    expectedVisString[0] = "(\"public\"&\"secret\"&\"topsecret\")|(\"confidential\"&\"topsecret\")";
    expectedVisString[1] = "(\"private\"&\"public\")|(\"private\"&\"topsecret\")|" + "(\"confidential\"&\"public\")|(\"confidential\"&\"topsecret\")";
    expectedVisString[2] = "(!\"topsecret\"&\"secret\")|(!\"topsecret\"&\"confidential\")";
    expectedVisString[3] = "(\"secret\"&\"" + COPYRIGHT + "\\\"" + ACCENT + "\\\\" + SECRET + "\\\"" + "\u0027&\\\\" + "\")";
    // setup configuration
    conf = HBaseConfiguration.create();
    conf.setInt("hfile.format.version", 3);
    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    conf.setInt("replication.source.size.capacity", 10240);
    conf.setLong("replication.source.sleepforretries", 100);
    conf.setInt("hbase.regionserver.maxlogs", 10);
    conf.setLong("hbase.master.logcleaner.ttl", 10);
    conf.setInt("zookeeper.recovery.retry", 1);
    conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
    conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
    conf.setInt("replication.stats.thread.period.seconds", 5);
    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    setVisibilityLabelServiceImpl(conf, ExpAsStringVisibilityLabelServiceImpl.class);
    conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
    VisibilityTestUtil.enableVisiblityLabels(conf);
    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, VisibilityReplication.class.getName());
    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, SimpleCP.class.getName());
    // Have to reset conf1 in case zk cluster location different
    // than default
    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
    conf.set("hbase.superuser", "admin");
    conf.set("hbase.superuser", User.getCurrent().getShortName());
    SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(), new String[] { "supergroup" });
    User.createUserForTesting(conf, User.getCurrent().getShortName(), new String[] { "supergroup" });
    USER1 = User.createUserForTesting(conf, "user1", new String[] {});
    TEST_UTIL = new HBaseTestingUtil(conf);
    TEST_UTIL.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
    zkw1 = new ZKWatcher(conf, "cluster1", null, true);
    // Base conf2 on conf1 so it gets the right zk cluster.
    conf1 = HBaseConfiguration.create(conf);
    conf1.setInt("hfile.format.version", 3);
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
    conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
    conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSink.class.getName());
    setVisibilityLabelServiceImpl(conf1, ExpAsStringVisibilityLabelServiceImpl.class);
    TEST_UTIL1 = new HBaseTestingUtil(conf1);
    TEST_UTIL1.setZkCluster(miniZK);
    zkw2 = new ZKWatcher(conf1, "cluster2", null, true);
    TEST_UTIL.startMiniCluster(1);
    // Wait for the labels table to become available
    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
    TEST_UTIL1.startMiniCluster(1);
    admin = TEST_UTIL.getAdmin();
    ReplicationPeerConfig rpc = ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL1.getClusterKey()).build();
    admin.addReplicationPeer("2", rpc);
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build();
    try (Admin hBaseAdmin = TEST_UTIL.getAdmin()) {
        hBaseAdmin.createTable(tableDescriptor);
    }
    try (Admin hBaseAdmin1 = TEST_UTIL1.getAdmin()) {
        hBaseAdmin1.createTable(tableDescriptor);
    }
    addLabels();
    setAuths(conf);
    setAuths(conf1);
}

Example 19 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestMasterCoprocessorExceptionWithRemove method testExceptionFromCoprocessorWhenCreatingTable.

@Test
public void testExceptionFromCoprocessorWhenCreatingTable() throws IOException {
    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
    HMaster master = cluster.getMaster();
    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
    BuggyMasterObserver cp = host.findCoprocessor(BuggyMasterObserver.class);
    assertFalse("No table created yet", cp.wasCreateTableCalled());
    // Set a watch on the zookeeper /hbase/master node. If the master dies,
    // the node will be deleted.
    // Master should *NOT* die:
    // we are testing that the default setting of hbase.coprocessor.abortonerror
    // =false
    // is respected.
    ZKWatcher zkw = new ZKWatcher(UTIL.getConfiguration(), "unittest", new Abortable() {

        @Override
        public void abort(String why, Throwable e) {
            throw new RuntimeException("Fatal ZK error: " + why, e);
        }

        @Override
        public boolean isAborted() {
            return false;
        }
    });
    MasterTracker masterTracker = new MasterTracker(zkw, "/hbase/master", new Abortable() {

        @Override
        public void abort(String why, Throwable e) {
            throw new RuntimeException("Fatal ZooKeeper tracker error, why=", e);
        }

        @Override
        public boolean isAborted() {
            return false;
        }
    });
    masterTracker.start();
    zkw.registerListener(masterTracker);
    // Test (part of the) output that should have be printed by master when it aborts:
    // (namely the part that shows the set of loaded coprocessors).
    // In this test, there is only a single coprocessor (BuggyMasterObserver).
    String coprocessorName = BuggyMasterObserver.class.getName();
    assertTrue(HMaster.getLoadedCoprocessors().contains(coprocessorName));
    TableDescriptor tableDescriptor1 = TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE1)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY1)).build();
    boolean threwDNRE = false;
    try {
        Admin admin = UTIL.getAdmin();
        admin.createTable(tableDescriptor1);
    } catch (IOException e) {
        if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
            threwDNRE = true;
        }
    } finally {
        assertTrue(threwDNRE);
    }
    // wait for a few seconds to make sure that the Master hasn't aborted.
    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        fail("InterruptedException while sleeping.");
    }
    assertFalse("Master survived coprocessor NPE, as expected.", masterTracker.masterZKNodeWasDeleted);
    String loadedCoprocessors = HMaster.getLoadedCoprocessors();
    assertTrue(loadedCoprocessors.contains(coprocessorName));
    // Verify that BuggyMasterObserver has been removed due to its misbehavior
    // by creating another table: should not have a problem this time.
    TableDescriptor tableDescriptor2 = TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE2)).setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY2)).build();
    Admin admin = UTIL.getAdmin();
    try {
        admin.createTable(tableDescriptor2);
    } catch (IOException e) {
        fail("Failed to create table after buggy coprocessor removal: " + e);
    }
}

Example 20 with ZKWatcher

use of org.apache.hadoop.hbase.zookeeper.ZKWatcher in project hbase by apache.

the class TestZooKeeperTableArchiveClient method setupCluster.

/**
 * Setup the config for the cluster
 */
@BeforeClass
public static void setupCluster() throws Exception {
    setupConf(UTIL.getConfiguration());
    UTIL.startMiniZKCluster();
    UTIL.getConfiguration().setClass(MockRegistry.REGISTRY_IMPL_CONF_KEY, MockRegistry.class, DummyConnectionRegistry.class);
    CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration());
    archivingClient = new ZKTableArchiveClient(UTIL.getConfiguration(), CONNECTION);
    // make hfile archiving node so we can archive files
    ZKWatcher watcher = UTIL.getZooKeeperWatcher();
    String archivingZNode = ZKTableArchiveClient.getArchiveZNode(UTIL.getConfiguration(), watcher);
    ZKUtil.createWithParents(watcher, archivingZNode);
    rss = mock(RegionServerServices.class);
    POOL = DirScanPool.getHFileCleanerScanPool(UTIL.getConfiguration());
}