Examples with ReplicationQueuesArguments org.apache.hadoop.hbase.replication.ReplicationQueuesArguments used on opensource projects

Example 6 with ReplicationQueuesArguments

use of org.apache.hadoop.hbase.replication.ReplicationQueuesArguments in project hbase by apache.

the class TestReplicationSourceManager method testCleanupFailoverQueues.

@Test
public void testCleanupFailoverQueues() throws Exception {
    final Server server = new DummyServer("hostname1.example.org");
    ReplicationQueues rq = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(server.getConfiguration(), server, server.getZooKeeper()));
    rq.init(server.getServerName().toString());
    // populate some znodes in the peer znode
    SortedSet<String> files = new TreeSet<>();
    String group = "testgroup";
    String file1 = group + ".log1";
    String file2 = group + ".log2";
    files.add(file1);
    files.add(file2);
    for (String file : files) {
        rq.addLog("1", file);
    }
    Server s1 = new DummyServer("dummyserver1.example.org");
    ReplicationQueues rq1 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1, s1.getZooKeeper()));
    rq1.init(s1.getServerName().toString());
    ReplicationPeers rp1 = ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration(), s1);
    rp1.init();
    NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName().getServerName(), rq1, rp1, new UUID(new Long(1), new Long(2)));
    w1.run();
    assertEquals(1, manager.getWalsByIdRecoveredQueues().size());
    String id = "1-" + server.getServerName().getServerName();
    assertEquals(files, manager.getWalsByIdRecoveredQueues().get(id).get(group));
    manager.cleanOldLogs(file2, id, true);
    // log1 should be deleted
    assertEquals(Sets.newHashSet(file2), manager.getWalsByIdRecoveredQueues().get(id).get(group));
}

Example 7 with ReplicationQueuesArguments

use of org.apache.hadoop.hbase.replication.ReplicationQueuesArguments in project hbase by apache.

the class TestReplicationSourceManager method testCleanupUnknownPeerZNode.

@Test
public void testCleanupUnknownPeerZNode() throws Exception {
    final Server server = new DummyServer("hostname2.example.org");
    ReplicationQueues rq = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(server.getConfiguration(), server, server.getZooKeeper()));
    rq.init(server.getServerName().toString());
    // populate some znodes in the peer znode
    // add log to an unknown peer
    String group = "testgroup";
    rq.addLog("2", group + ".log1");
    rq.addLog("2", group + ".log2");
    NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName().getServerName());
    w1.run();
    // The log of the unknown peer should be removed from zk
    for (String peer : manager.getAllQueues()) {
        assertTrue(peer.startsWith("1"));
    }
}

Example 8 with ReplicationQueuesArguments

use of org.apache.hadoop.hbase.replication.ReplicationQueuesArguments in project hbase by apache.

the class TestReplicationSourceManagerZkImpl method testFailoverDeadServerCversionChange.

@Test
public void testFailoverDeadServerCversionChange() throws Exception {
    final Server s0 = new DummyServer("cversion-change0.example.org");
    ReplicationQueues repQueues = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, s0, s0.getZooKeeper()));
    repQueues.init(s0.getServerName().toString());
    // populate some znodes in the peer znode
    files.add("log1");
    files.add("log2");
    for (String file : files) {
        repQueues.addLog("1", file);
    }
    // simulate queue transfer
    Server s1 = new DummyServer("cversion-change1.example.org");
    ReplicationQueues rq1 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1, s1.getZooKeeper()));
    rq1.init(s1.getServerName().toString());
    ReplicationQueuesClientZKImpl client = (ReplicationQueuesClientZKImpl) ReplicationFactory.getReplicationQueuesClient(new ReplicationQueuesClientArguments(s1.getConfiguration(), s1, s1.getZooKeeper()));
    int v0 = client.getQueuesZNodeCversion();
    List<String> queues = rq1.getUnClaimedQueueIds(s0.getServerName().getServerName());
    for (String queue : queues) {
        rq1.claimQueue(s0.getServerName().getServerName(), queue);
    }
    rq1.removeReplicatorIfQueueIsEmpty(s0.getServerName().getServerName());
    int v1 = client.getQueuesZNodeCversion();
    // cversion should increase by 1 since a child node is deleted
    assertEquals(v0 + 1, v1);
    s0.stop("");
}

Example 9 with ReplicationQueuesArguments

use of org.apache.hadoop.hbase.replication.ReplicationQueuesArguments in project hbase by apache.

the class TestReplicationSourceManagerZkImpl method testNodeFailoverDeadServerParsing.

// Tests the naming convention of adopted queues for ReplicationQueuesZkImpl
@Test
public void testNodeFailoverDeadServerParsing() throws Exception {
    final Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
    ReplicationQueues repQueues = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper()));
    repQueues.init(server.getServerName().toString());
    // populate some znodes in the peer znode
    files.add("log1");
    files.add("log2");
    for (String file : files) {
        repQueues.addLog("1", file);
    }
    // create 3 DummyServers
    Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal");
    Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com");
    Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
    // simulate three servers fail sequentially
    ReplicationQueues rq1 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1, s1.getZooKeeper()));
    rq1.init(s1.getServerName().toString());
    String serverName = server.getServerName().getServerName();
    List<String> unclaimed = rq1.getUnClaimedQueueIds(serverName);
    rq1.claimQueue(serverName, unclaimed.get(0)).getSecond();
    rq1.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
    ReplicationQueues rq2 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s2.getConfiguration(), s2, s2.getZooKeeper()));
    rq2.init(s2.getServerName().toString());
    serverName = s1.getServerName().getServerName();
    unclaimed = rq2.getUnClaimedQueueIds(serverName);
    rq2.claimQueue(serverName, unclaimed.get(0)).getSecond();
    rq2.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
    ReplicationQueues rq3 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s3.getConfiguration(), s3, s3.getZooKeeper()));
    rq3.init(s3.getServerName().toString());
    serverName = s2.getServerName().getServerName();
    unclaimed = rq3.getUnClaimedQueueIds(serverName);
    String queue3 = rq3.claimQueue(serverName, unclaimed.get(0)).getFirst();
    rq3.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
    ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queue3);
    List<String> result = replicationQueueInfo.getDeadRegionServers();
    // verify
    assertTrue(result.contains(server.getServerName().getServerName()));
    assertTrue(result.contains(s1.getServerName().getServerName()));
    assertTrue(result.contains(s2.getServerName().getServerName()));
    server.stop("");
}

Example 10 with ReplicationQueuesArguments

use of org.apache.hadoop.hbase.replication.ReplicationQueuesArguments in project hbase by apache.

the class Replication method initialize.

public void initialize(final Server server, final FileSystem fs, final Path logDir, final Path oldLogDir) throws IOException {
    this.server = server;
    this.conf = this.server.getConfiguration();
    this.replicationForBulkLoadData = isReplicationForBulkLoadDataEnabled(this.conf);
    this.scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(server.getServerName().toShortString() + "Replication Statistics #%d").setDaemon(true).build());
    if (this.replicationForBulkLoadData) {
        if (conf.get(HConstants.REPLICATION_CLUSTER_ID) == null || conf.get(HConstants.REPLICATION_CLUSTER_ID).isEmpty()) {
            throw new IllegalArgumentException(HConstants.REPLICATION_CLUSTER_ID + " cannot be null/empty when " + HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is set to true.");
        }
    }
    try {
        this.replicationQueues = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, this.server, server.getZooKeeper()));
        this.replicationQueues.init(this.server.getServerName().toString());
        this.replicationPeers = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf, this.server);
        this.replicationPeers.init();
        this.replicationTracker = ReplicationFactory.getReplicationTracker(server.getZooKeeper(), this.replicationPeers, this.conf, this.server, this.server);
    } catch (Exception e) {
        throw new IOException("Failed replication handler create", e);
    }
    UUID clusterId = null;
    try {
        clusterId = ZKClusterId.getUUIDForCluster(this.server.getZooKeeper());
    } catch (KeeperException ke) {
        throw new IOException("Could not read cluster id", ke);
    }
    this.replicationManager = new ReplicationSourceManager(replicationQueues, replicationPeers, replicationTracker, conf, this.server, fs, logDir, oldLogDir, clusterId);
    this.statsThreadPeriod = this.conf.getInt("replication.stats.thread.period.seconds", 5 * 60);
    LOG.debug("ReplicationStatisticsThread " + this.statsThreadPeriod);
    this.replicationLoad = new ReplicationLoad();
}