Examples with ReplicationPeerConfig org.apache.hadoop.hbase.replication.ReplicationPeerConfig used on opensource projects

Example 11 with ReplicationPeerConfig

use of org.apache.hadoop.hbase.replication.ReplicationPeerConfig in project hbase by apache.

the class TestTableCFsUpdater method testUpgrade.

@Test
public void testUpgrade() throws KeeperException, InterruptedException, DeserializationException {
    String peerId = "1";
    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
    final TableName tableName3 = TableName.valueOf(name.getMethodName() + "3");
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(zkw.getQuorum());
    String peerNode = getPeerNode(peerId);
    ZKUtil.createWithParents(zkw, peerNode, ReplicationSerDeHelper.toByteArray(rpc));
    String tableCFs = tableName1 + ":cf1,cf2;" + tableName2 + ":cf3;" + tableName3;
    String tableCFsNode = getTableCFsNode(peerId);
    LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
    ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
    ReplicationPeerConfig actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    String actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    assertNull(actualRpc.getTableCFsMap());
    assertEquals(tableCFs, actualTableCfs);
    peerId = "2";
    rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(zkw.getQuorum());
    peerNode = getPeerNode(peerId);
    ZKUtil.createWithParents(zkw, peerNode, ReplicationSerDeHelper.toByteArray(rpc));
    tableCFs = tableName1 + ":cf1,cf3;" + tableName2 + ":cf2";
    tableCFsNode = getTableCFsNode(peerId);
    LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
    ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    assertNull(actualRpc.getTableCFsMap());
    assertEquals(tableCFs, actualTableCfs);
    peerId = "3";
    rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(zkw.getQuorum());
    peerNode = getPeerNode(peerId);
    ZKUtil.createWithParents(zkw, peerNode, ReplicationSerDeHelper.toByteArray(rpc));
    tableCFs = "";
    tableCFsNode = getTableCFsNode(peerId);
    LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
    ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    assertNull(actualRpc.getTableCFsMap());
    assertEquals(tableCFs, actualTableCfs);
    peerId = "4";
    rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(zkw.getQuorum());
    peerNode = getPeerNode(peerId);
    ZKUtil.createWithParents(zkw, peerNode, ReplicationSerDeHelper.toByteArray(rpc));
    tableCFsNode = getTableCFsNode(peerId);
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    assertNull(actualRpc.getTableCFsMap());
    assertNull(actualTableCfs);
    update();
    peerId = "1";
    peerNode = getPeerNode(peerId);
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    Map<TableName, List<String>> tableNameListMap = actualRpc.getTableCFsMap();
    assertEquals(3, tableNameListMap.size());
    assertTrue(tableNameListMap.containsKey(tableName1));
    assertTrue(tableNameListMap.containsKey(tableName2));
    assertTrue(tableNameListMap.containsKey(tableName3));
    assertEquals(2, tableNameListMap.get(tableName1).size());
    assertEquals("cf1", tableNameListMap.get(tableName1).get(0));
    assertEquals("cf2", tableNameListMap.get(tableName1).get(1));
    assertEquals(1, tableNameListMap.get(tableName2).size());
    assertEquals("cf3", tableNameListMap.get(tableName2).get(0));
    assertNull(tableNameListMap.get(tableName3));
    peerId = "2";
    peerNode = getPeerNode(peerId);
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    tableNameListMap = actualRpc.getTableCFsMap();
    assertEquals(2, tableNameListMap.size());
    assertTrue(tableNameListMap.containsKey(tableName1));
    assertTrue(tableNameListMap.containsKey(tableName2));
    assertEquals(2, tableNameListMap.get(tableName1).size());
    assertEquals("cf1", tableNameListMap.get(tableName1).get(0));
    assertEquals("cf3", tableNameListMap.get(tableName1).get(1));
    assertEquals(1, tableNameListMap.get(tableName2).size());
    assertEquals("cf2", tableNameListMap.get(tableName2).get(0));
    peerId = "3";
    peerNode = getPeerNode(peerId);
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    tableNameListMap = actualRpc.getTableCFsMap();
    assertNull(tableNameListMap);
    peerId = "4";
    peerNode = getPeerNode(peerId);
    actualRpc = ReplicationSerDeHelper.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
    assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
    tableNameListMap = actualRpc.getTableCFsMap();
    assertNull(tableNameListMap);
}

Example 12 with ReplicationPeerConfig

use of org.apache.hadoop.hbase.replication.ReplicationPeerConfig in project hbase by apache.

the class TestGlobalThrottler method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    conf1 = HBaseConfiguration.create();
    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
    conf1.setLong("replication.source.sleepforretries", 100);
    // Each WAL is about 120 bytes
    conf1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, 200);
    conf1.setLong("replication.source.per.peer.node.bandwidth", 100L);
    utility1 = new HBaseTestingUtility(conf1);
    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    new ZooKeeperWatcher(conf1, "cluster1", null, true);
    conf2 = new Configuration(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
    utility2 = new HBaseTestingUtility(conf2);
    utility2.setZkCluster(miniZK);
    new ZooKeeperWatcher(conf2, "cluster2", null, true);
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationPeerConfig rpc = new ReplicationPeerConfig();
    rpc.setClusterKey(utility2.getClusterKey());
    utility1.startMiniCluster(1, 1);
    utility2.startMiniCluster(1, 1);
    admin1.addPeer("peer1", rpc, null);
    admin1.addPeer("peer2", rpc, null);
    admin1.addPeer("peer3", rpc, null);
}

Example 13 with ReplicationPeerConfig

use of org.apache.hadoop.hbase.replication.ReplicationPeerConfig in project hbase by apache.

the class HBaseAdmin method getPeerClusterConfiguration.

/**
   * Returns the configuration needed to talk to the remote slave cluster.
   * @param peer the description of replication peer
   * @return the configuration for the peer cluster, null if it was unable to get the configuration
   * @throws IOException
   */
private Configuration getPeerClusterConfiguration(ReplicationPeerDescription peer) throws IOException {
    ReplicationPeerConfig peerConfig = peer.getPeerConfig();
    Configuration otherConf;
    try {
        otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
    } catch (IOException e) {
        throw new IOException("Can't get peer configuration for peerId=" + peer.getPeerId(), e);
    }
    if (!peerConfig.getConfiguration().isEmpty()) {
        CompoundConfiguration compound = new CompoundConfiguration();
        compound.add(otherConf);
        compound.addStringMap(peerConfig.getConfiguration());
        return compound;
    }
    return otherConf;
}

Example 14 with ReplicationPeerConfig

use of org.apache.hadoop.hbase.replication.ReplicationPeerConfig in project hbase by apache.

the class ReplicationAdmin method appendPeerTableCFs.

/**
   * Append the replicable table-cf config of the specified peer
   * @param id a short that identifies the cluster
   * @param tableCfs A map from tableName to column family names
   * @throws ReplicationException
   * @throws IOException
   */
@Deprecated
public void appendPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs) throws ReplicationException, IOException {
    if (tableCfs == null) {
        throw new ReplicationException("tableCfs is null");
    }
    ReplicationPeerConfig peerConfig = admin.getReplicationPeerConfig(id);
    Map<TableName, List<String>> preTableCfs = peerConfig.getTableCFsMap();
    if (preTableCfs == null) {
        setPeerTableCFs(id, tableCfs);
        return;
    }
    for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
        TableName table = entry.getKey();
        Collection<String> appendCfs = entry.getValue();
        if (preTableCfs.containsKey(table)) {
            List<String> cfs = preTableCfs.get(table);
            if (cfs == null || appendCfs == null || appendCfs.isEmpty()) {
                preTableCfs.put(table, null);
            } else {
                Set<String> cfSet = new HashSet<>(cfs);
                cfSet.addAll(appendCfs);
                preTableCfs.put(table, Lists.newArrayList(cfSet));
            }
        } else {
            if (appendCfs == null || appendCfs.isEmpty()) {
                preTableCfs.put(table, null);
            } else {
                preTableCfs.put(table, Lists.newArrayList(appendCfs));
            }
        }
    }
    updatePeerConfig(id, peerConfig);
}

Example 15 with ReplicationPeerConfig

use of org.apache.hadoop.hbase.replication.ReplicationPeerConfig in project hbase by apache.

the class ReplicationAdmin method setPeerTableCFs.

/**
   * Set the replicable table-cf config of the specified peer
   * @param id a short name that identifies the cluster
   * @param tableCfs the table and column-family list which will be replicated for this peer.
   * A map from tableName to column family names. An empty collection can be passed
   * to indicate replicating all column families. Pass null for replicating all table and column
   * families
   */
@Deprecated
public void setPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs) throws IOException {
    ReplicationPeerConfig peerConfig = getPeerConfig(id);
    peerConfig.setTableCFsMap(tableCfs);
    updatePeerConfig(id, peerConfig);
}