Examples with ReplicationPeer org.apache.hadoop.hbase.replication.ReplicationPeer used on opensource projects

Example 6 with ReplicationPeer

use of org.apache.hadoop.hbase.replication.ReplicationPeer in project hbase by apache.

the class ReplicationSourceManager method drainSources.

/**
 * <p>
 * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.
 * </p>
 * <p>
 * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal
 * files for a replication peer as we do not need to replicate them any more. And this is
 * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}
 * later, the stale data will be replicated again and cause inconsistency.
 * </p>
 * <p>
 * See HBASE-20426 for more details.
 * </p>
 * @param peerId the id of the sync replication peer
 */
public void drainSources(String peerId) throws IOException, ReplicationException {
    String terminateMessage = "Sync replication peer " + peerId + " is transiting to STANDBY. Will close the previous replication source and open a new one";
    ReplicationPeer peer = replicationPeers.getPeer(peerId);
    assert peer.getPeerConfig().isSyncReplication();
    ReplicationSourceInterface src = createSource(peerId, peer);
    // synchronized here to avoid race with preLogRoll where we add new log to source and also
    // walsById.
    ReplicationSourceInterface toRemove;
    Map<String, NavigableSet<String>> wals = new HashMap<>();
    synchronized (latestPaths) {
        toRemove = sources.put(peerId, src);
        if (toRemove != null) {
            LOG.info("Terminate replication source for " + toRemove.getPeerId());
            toRemove.terminate(terminateMessage);
            toRemove.getSourceMetrics().clear();
        }
        // Here we make a copy of all the remaining wal files and then delete them from the
        // replication queue storage after releasing the lock. It is not safe to just remove the old
        // map from walsById since later we may fail to delete them from the replication queue
        // storage, and when we retry next time, we can not know the wal files that need to be deleted
        // from the replication queue storage.
        walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
    }
    LOG.info("Startup replication source for " + src.getPeerId());
    src.startup();
    for (NavigableSet<String> walsByGroup : wals.values()) {
        for (String wal : walsByGroup) {
            queueStorage.removeWAL(server.getServerName(), peerId, wal);
        }
    }
    synchronized (walsById) {
        Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);
        wals.forEach((k, v) -> {
            NavigableSet<String> walsByGroup = oldWals.get(k);
            if (walsByGroup != null) {
                walsByGroup.removeAll(v);
            }
        });
    }
    // simplify the logic.
    synchronized (this.oldsources) {
        for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext(); ) {
            ReplicationSourceInterface oldSource = iter.next();
            if (oldSource.getPeerId().equals(peerId)) {
                String queueId = oldSource.getQueueId();
                oldSource.terminate(terminateMessage);
                oldSource.getSourceMetrics().clear();
                queueStorage.removeQueue(server.getServerName(), queueId);
                walsByIdRecoveredQueues.remove(queueId);
                iter.remove();
            }
        }
    }
}

Example 7 with ReplicationPeer

use of org.apache.hadoop.hbase.replication.ReplicationPeer in project hbase by apache.

the class ReplicationSourceManager method refreshSources.

/**
 * Close the previous replication sources of this peer id and open new sources to trigger the new
 * replication state changes or new replication config changes. Here we don't need to change
 * replication queue storage and only to enqueue all logs to the new replication source
 * @param peerId the id of the replication peer
 */
public void refreshSources(String peerId) throws IOException {
    String terminateMessage = "Peer " + peerId + " state or config changed. Will close the previous replication source and open a new one";
    ReplicationPeer peer = replicationPeers.getPeer(peerId);
    ReplicationSourceInterface src = createSource(peerId, peer);
    // synchronized on latestPaths to avoid missing the new log
    synchronized (this.latestPaths) {
        ReplicationSourceInterface toRemove = this.sources.put(peerId, src);
        if (toRemove != null) {
            LOG.info("Terminate replication source for " + toRemove.getPeerId());
            // Do not clear metrics
            toRemove.terminate(terminateMessage, null, false);
        }
        for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {
            walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));
        }
    }
    LOG.info("Startup replication source for " + src.getPeerId());
    src.startup();
    List<ReplicationSourceInterface> toStartup = new ArrayList<>();
    // synchronized on oldsources to avoid race with NodeFailoverWorker
    synchronized (this.oldsources) {
        List<String> previousQueueIds = new ArrayList<>();
        for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter.hasNext(); ) {
            ReplicationSourceInterface oldSource = iter.next();
            if (oldSource.getPeerId().equals(peerId)) {
                previousQueueIds.add(oldSource.getQueueId());
                oldSource.terminate(terminateMessage);
                iter.remove();
            }
        }
        for (String queueId : previousQueueIds) {
            ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);
            this.oldsources.add(recoveredReplicationSource);
            for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {
                walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));
            }
            toStartup.add(recoveredReplicationSource);
        }
    }
    for (ReplicationSourceInterface replicationSource : toStartup) {
        replicationSource.startup();
    }
}

Example 8 with ReplicationPeer

use of org.apache.hadoop.hbase.replication.ReplicationPeer in project hbase by apache.

the class ReplicationAdmin method listReplicationPeers.

/**
   * @deprecated use {@link org.apache.hadoop.hbase.client.Admin#listReplicationPeers()} instead
   */
@VisibleForTesting
@Deprecated
List<ReplicationPeer> listReplicationPeers() throws IOException {
    Map<String, ReplicationPeerConfig> peers = listPeerConfigs();
    if (peers == null || peers.size() <= 0) {
        return null;
    }
    List<ReplicationPeer> listOfPeers = new ArrayList<>(peers.size());
    for (Entry<String, ReplicationPeerConfig> peerEntry : peers.entrySet()) {
        String peerId = peerEntry.getKey();
        try {
            Pair<ReplicationPeerConfig, Configuration> pair = this.replicationPeers.getPeerConf(peerId);
            Configuration peerConf = pair.getSecond();
            ReplicationPeer peer = new ReplicationPeerZKImpl(zkw, pair.getSecond(), peerId, pair.getFirst(), this.connection);
            listOfPeers.add(peer);
        } catch (ReplicationException e) {
            LOG.warn("Failed to get valid replication peers. " + "Error connecting to peer cluster with peerId=" + peerId + ". Error message=" + e.getMessage());
            LOG.debug("Failure details to get valid replication peers.", e);
            continue;
        }
    }
    return listOfPeers;
}

Example 9 with ReplicationPeer

use of org.apache.hadoop.hbase.replication.ReplicationPeer in project hbase by apache.

the class ReplicationSourceManager method removePeer.

/**
 * <ol>
 * <li>Remove peer for replicationPeers</li>
 * <li>Remove all the recovered sources for the specified id and related replication queues</li>
 * <li>Remove the normal source and related replication queue</li>
 * <li>Remove HFile Refs</li>
 * </ol>
 * @param peerId the id of the replication peer
 */
public void removePeer(String peerId) {
    ReplicationPeer peer = replicationPeers.removePeer(peerId);
    String terminateMessage = "Replication stream was removed by a user";
    List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();
    // see NodeFailoverWorker.run
    synchronized (this.oldsources) {
        // First close all the recovered sources for this peer
        for (ReplicationSourceInterface src : oldsources) {
            if (peerId.equals(src.getPeerId())) {
                oldSourcesToDelete.add(src);
            }
        }
        for (ReplicationSourceInterface src : oldSourcesToDelete) {
            src.terminate(terminateMessage);
            removeRecoveredSource(src);
        }
    }
    LOG.info("Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());
    // Now close the normal source for this peer
    ReplicationSourceInterface srcToRemove = this.sources.get(peerId);
    if (srcToRemove != null) {
        srcToRemove.terminate(terminateMessage);
        removeSource(srcToRemove);
    } else {
        // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup
        // Delete queue from storage and memory and queue id is same with peer id for normal
        // source
        deleteQueue(peerId);
        this.walsById.remove(peerId);
    }
    ReplicationPeerConfig peerConfig = peer.getPeerConfig();
    if (peerConfig.isSyncReplication()) {
        syncReplicationPeerMappingManager.remove(peerId, peerConfig);
    }
    // Remove HFile Refs
    abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));
}

Example 10 with ReplicationPeer

use of org.apache.hadoop.hbase.replication.ReplicationPeer in project hbase by apache.

the class ReplicationSourceManager method addSource.

/**
 * Add a normal source for the given peer on this region server. Meanwhile, add new replication
 * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal
 * group and do replication
 * @param peerId the id of the replication peer
 * @return the source that was created
 */
void addSource(String peerId) throws IOException {
    ReplicationPeer peer = replicationPeers.getPeer(peerId);
    if (ReplicationUtils.LEGACY_REGION_REPLICATION_ENDPOINT_NAME.equals(peer.getPeerConfig().getReplicationEndpointImpl())) {
        // we do not use this endpoint for region replication any more, see HBASE-26233
        LOG.info("Legacy region replication peer found, skip adding: {}", peer.getPeerConfig());
        return;
    }
    ReplicationSourceInterface src = createSource(peerId, peer);
    // synchronized on latestPaths to avoid missing the new log
    synchronized (this.latestPaths) {
        this.sources.put(peerId, src);
        Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
        this.walsById.put(peerId, walsByGroup);
        // Add the latest wal to that source's queue
        if (!latestPaths.isEmpty()) {
            for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {
                Path walPath = walPrefixAndPath.getValue();
                NavigableSet<String> wals = new TreeSet<>();
                wals.add(walPath.getName());
                walsByGroup.put(walPrefixAndPath.getKey(), wals);
                // Abort RS and throw exception to make add peer failed
                abortAndThrowIOExceptionWhenFail(() -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));
                src.enqueueLog(walPath);
                LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());
            }
        }
    }
    ReplicationPeerConfig peerConfig = peer.getPeerConfig();
    if (peerConfig.isSyncReplication()) {
        syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);
    }
    src.startup();
}