Examples with WALProvider org.apache.hadoop.hbase.wal.WALProvider used on opensource projects

Example 1 with WALProvider

use of org.apache.hadoop.hbase.wal.WALProvider in project hbase by apache.

the class Replication method initialize.

@Override
public void initialize(Server server, FileSystem fs, Path logDir, Path oldLogDir, WALFactory walFactory) throws IOException {
    this.server = server;
    this.conf = this.server.getConfiguration();
    this.isReplicationForBulkLoadDataEnabled = ReplicationUtils.isReplicationForBulkLoadDataEnabled(this.conf);
    if (this.isReplicationForBulkLoadDataEnabled) {
        if (conf.get(HConstants.REPLICATION_CLUSTER_ID) == null || conf.get(HConstants.REPLICATION_CLUSTER_ID).isEmpty()) {
            throw new IllegalArgumentException(HConstants.REPLICATION_CLUSTER_ID + " cannot be null/empty when " + HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is set to true.");
        }
    }
    try {
        this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
        this.replicationPeers = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf);
        this.replicationPeers.init();
    } catch (Exception e) {
        throw new IOException("Failed replication handler create", e);
    }
    UUID clusterId = null;
    try {
        clusterId = ZKClusterId.getUUIDForCluster(this.server.getZooKeeper());
    } catch (KeeperException ke) {
        throw new IOException("Could not read cluster id", ke);
    }
    SyncReplicationPeerMappingManager mapping = new SyncReplicationPeerMappingManager();
    this.globalMetricsSource = CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class).getGlobalSource();
    this.replicationManager = new ReplicationSourceManager(queueStorage, replicationPeers, conf, this.server, fs, logDir, oldLogDir, clusterId, walFactory, mapping, globalMetricsSource);
    this.syncReplicationPeerInfoProvider = new SyncReplicationPeerInfoProviderImpl(replicationPeers, mapping);
    PeerActionListener peerActionListener = PeerActionListener.DUMMY;
    // Get the user-space WAL provider
    WALProvider walProvider = walFactory != null ? walFactory.getWALProvider() : null;
    if (walProvider != null) {
        walProvider.addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
        if (walProvider instanceof SyncReplicationWALProvider) {
            SyncReplicationWALProvider syncWALProvider = (SyncReplicationWALProvider) walProvider;
            peerActionListener = syncWALProvider;
            syncWALProvider.setPeerInfoProvider(syncReplicationPeerInfoProvider);
            // for sync replication state change, we need to reload the state twice, you can see the
            // code in PeerProcedureHandlerImpl, so here we need to go over the sync replication peers
            // to see if any of them are in the middle of the two refreshes, if so, we need to manually
            // repeat the action we have done in the first refresh, otherwise when the second refresh
            // comes we will be in trouble, such as NPE.
            replicationPeers.getAllPeerIds().stream().map(replicationPeers::getPeer).filter(p -> p.getPeerConfig().isSyncReplication()).filter(p -> p.getNewSyncReplicationState() != SyncReplicationState.NONE).forEach(p -> syncWALProvider.peerSyncReplicationStateChange(p.getId(), p.getSyncReplicationState(), p.getNewSyncReplicationState(), 0));
        }
    }
    this.statsPeriodInSecond = this.conf.getInt("replication.stats.thread.period.seconds", 5 * 60);
    this.replicationLoad = new ReplicationLoad();
    this.peerProcedureHandler = new PeerProcedureHandlerImpl(replicationManager, peerActionListener);
}

Example 2 with WALProvider

use of org.apache.hadoop.hbase.wal.WALProvider in project hbase by apache.

the class TestWALConfiguration method testBlocksizeDefaultsToTwiceHDFSBlockSize.

/**
 * Test blocksize change from HBASE-20520 takes on both asycnfs and old wal provider.
 * Hard to verify more than this given the blocksize is passed down to HDFS on create -- not
 * kept local to the streams themselves.
 */
@Test
public void testBlocksizeDefaultsToTwiceHDFSBlockSize() throws IOException {
    TableName tableName = TableName.valueOf("test");
    final WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), this.walProvider);
    Configuration conf = TEST_UTIL.getConfiguration();
    WALProvider provider = walFactory.getWALProvider();
    // Get a WAL instance from the provider. Check its blocksize.
    WAL wal = provider.getWAL(null);
    if (wal instanceof AbstractFSWAL) {
        long expectedDefaultBlockSize = WALUtil.getWALBlockSize(conf, FileSystem.get(conf), TEST_UTIL.getDataTestDir());
        long blocksize = ((AbstractFSWAL) wal).blocksize;
        assertEquals(expectedDefaultBlockSize, blocksize);
        LOG.info("Found blocksize of {} on {}", blocksize, wal);
    } else {
        fail("Unknown provider " + provider);
    }
}