use of org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy in project distributedlog by twitter.
the class BKDistributedLogNamespace method createBKZKClientBuilder.
private static ZooKeeperClientBuilder createBKZKClientBuilder(String zkcName, DistributedLogConfiguration conf, String zkServers, StatsLogger statsLogger) {
RetryPolicy retryPolicy = null;
if (conf.getZKNumRetries() > 0) {
retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getBKClientZKRetryBackoffStartMillis(), conf.getBKClientZKRetryBackoffMaxMillis(), conf.getBKClientZKNumRetries());
}
ZooKeeperClientBuilder builder = ZooKeeperClientBuilder.newBuilder().name(zkcName).sessionTimeoutMs(conf.getBKClientZKSessionTimeoutMilliSeconds()).retryThreadCount(conf.getZKClientNumberRetryThreads()).requestRateLimit(conf.getBKClientZKRequestRateLimit()).zkServers(zkServers).retryPolicy(retryPolicy).statsLogger(statsLogger).zkAclId(conf.getZkAclId());
LOG.info("Created shared zooKeeper client builder {}: zkServers = {}, numRetries = {}, sessionTimeout = {}, retryBackoff = {}," + " maxRetryBackoff = {}, zkAclId = {}.", new Object[] { zkcName, zkServers, conf.getBKClientZKNumRetries(), conf.getBKClientZKSessionTimeoutMilliSeconds(), conf.getBKClientZKRetryBackoffStartMillis(), conf.getBKClientZKRetryBackoffMaxMillis(), conf.getZkAclId() });
return builder;
}
use of org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy in project distributedlog by twitter.
the class DLAuditor method calculateLedgerSpaceUsage.
public long calculateLedgerSpaceUsage(URI uri) throws IOException {
List<URI> uris = Lists.newArrayList(uri);
String zkServers = validateAndGetZKServers(uris);
RetryPolicy retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getZKRetryBackoffStartMillis(), conf.getZKRetryBackoffMaxMillis(), Integer.MAX_VALUE);
ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().name("DLAuditor-ZK").zkServers(zkServers).sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).retryPolicy(retryPolicy).zkAclId(conf.getZkAclId()).build();
ExecutorService executorService = Executors.newCachedThreadPool();
try {
BKDLConfig bkdlConfig = resolveBKDLConfig(zkc, uris);
logger.info("Resolved bookkeeper config : {}", bkdlConfig);
BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder().name("DLAuditor-BK").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).build();
try {
return calculateLedgerSpaceUsage(bkc, executorService);
} finally {
bkc.close();
}
} finally {
zkc.close();
executorService.shutdown();
}
}
use of org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy in project distributedlog by twitter.
the class DLAuditor method collectLedgers.
public Pair<Set<Long>, Set<Long>> collectLedgers(List<URI> uris, List<List<String>> allocationPaths) throws IOException {
Preconditions.checkArgument(uris.size() > 0, "No uri provided to audit");
String zkServers = validateAndGetZKServers(uris);
RetryPolicy retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getZKRetryBackoffStartMillis(), conf.getZKRetryBackoffMaxMillis(), Integer.MAX_VALUE);
ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().name("DLAuditor-ZK").zkServers(zkServers).sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).retryPolicy(retryPolicy).zkAclId(conf.getZkAclId()).build();
ExecutorService executorService = Executors.newCachedThreadPool();
try {
BKDLConfig bkdlConfig = resolveBKDLConfig(zkc, uris);
logger.info("Resolved bookkeeper config : {}", bkdlConfig);
BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder().name("DLAuditor-BK").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).build();
try {
Set<Long> bkLedgers = collectLedgersFromBK(bkc, executorService);
Set<Long> dlLedgers = collectLedgersFromDL(uris, allocationPaths);
return Pair.of(bkLedgers, dlLedgers);
} finally {
bkc.close();
}
} finally {
zkc.close();
executorService.shutdown();
}
}
use of org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy in project distributedlog by twitter.
the class ZooKeeperClient method buildZooKeeper.
private ZooKeeper buildZooKeeper() throws ZooKeeperConnectionException, InterruptedException {
Watcher watcher = new Watcher() {
@Override
public void process(WatchedEvent event) {
switch(event.getType()) {
case None:
switch(event.getState()) {
case Expired:
if (null == retryPolicy) {
LOG.info("ZooKeeper {}' session expired. Event: {}", name, event);
closeInternal();
}
authenticated = false;
break;
case Disconnected:
if (null == retryPolicy) {
LOG.info("ZooKeeper {} is disconnected from zookeeper now," + " but it is OK unless we received EXPIRED event.", name);
}
// Mark as not authenticated if expired or disconnected. In both cases
// we lose any attached auth info. Relying on Expired/Disconnected is
// sufficient since all Expired/Disconnected events are processed before
// all SyncConnected events, and the underlying member is not updated until
// SyncConnected is received.
authenticated = false;
break;
default:
break;
}
}
try {
for (Watcher watcher : watchers) {
try {
watcher.process(event);
} catch (Throwable t) {
LOG.warn("Encountered unexpected exception from watcher {} : ", watcher, t);
}
}
} catch (Throwable t) {
LOG.warn("Encountered unexpected exception when firing watched event {} : ", event, t);
}
}
};
Set<Watcher> watchers = new HashSet<Watcher>();
watchers.add(watcher);
ZooKeeper zk;
try {
RetryPolicy opRetryPolicy = null == retryPolicy ? new BoundExponentialBackoffRetryPolicy(sessionTimeoutMs, sessionTimeoutMs, 0) : retryPolicy;
RetryPolicy connectRetryPolicy = null == retryPolicy ? new BoundExponentialBackoffRetryPolicy(sessionTimeoutMs, sessionTimeoutMs, 0) : new BoundExponentialBackoffRetryPolicy(sessionTimeoutMs, sessionTimeoutMs, Integer.MAX_VALUE);
zk = org.apache.bookkeeper.zookeeper.ZooKeeperClient.newBuilder().connectString(zooKeeperServers).sessionTimeoutMs(sessionTimeoutMs).watchers(watchers).operationRetryPolicy(opRetryPolicy).connectRetryPolicy(connectRetryPolicy).statsLogger(statsLogger).retryThreadCount(retryThreadCount).requestRateLimit(requestRateLimit).build();
} catch (KeeperException e) {
throw new ZooKeeperConnectionException("Problem connecting to servers: " + zooKeeperServers, e);
} catch (IOException e) {
throw new ZooKeeperConnectionException("Problem connecting to servers: " + zooKeeperServers, e);
}
return zk;
}
use of org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy in project distributedlog by twitter.
the class BookKeeperClient method initialize.
private synchronized void initialize() throws IOException {
if (null != this.bkc) {
return;
}
boolean registerExpirationHandler;
if (null == this.zkc) {
int zkSessionTimeout = conf.getBKClientZKSessionTimeoutMilliSeconds();
RetryPolicy retryPolicy = null;
if (conf.getBKClientZKNumRetries() > 0) {
retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getBKClientZKRetryBackoffStartMillis(), conf.getBKClientZKRetryBackoffMaxMillis(), conf.getBKClientZKNumRetries());
}
Credentials credentials = Credentials.NONE;
if (conf.getZkAclId() != null) {
credentials = new DigestCredentials(conf.getZkAclId(), conf.getZkAclId());
}
this.zkc = new ZooKeeperClient(name + ":zk", zkSessionTimeout, 2 * zkSessionTimeout, zkServers, retryPolicy, statsLogger.scope("bkc_zkc"), conf.getZKClientNumberRetryThreads(), conf.getBKClientZKRequestRateLimit(), credentials);
}
registerExpirationHandler = conf.getBKClientZKNumRetries() <= 0;
try {
commonInitialization(conf, ledgersPath, channelFactory, statsLogger, requestTimer, registerExpirationHandler);
} catch (InterruptedException e) {
throw new DLInterruptedException("Interrupted on creating bookkeeper client " + name + " : ", e);
} catch (KeeperException e) {
throw new ZKException("Error on creating bookkeeper client " + name + " : ", e);
}
if (ownZK) {
LOG.info("BookKeeper Client created {} with its own ZK Client : ledgersPath = {}, numRetries = {}, " + "sessionTimeout = {}, backoff = {}, maxBackoff = {}, dnsResolver = {}, registerExpirationHandler = {}", new Object[] { name, ledgersPath, conf.getBKClientZKNumRetries(), conf.getBKClientZKSessionTimeoutMilliSeconds(), conf.getBKClientZKRetryBackoffStartMillis(), conf.getBKClientZKRetryBackoffMaxMillis(), conf.getBkDNSResolverOverrides(), registerExpirationHandler });
} else {
LOG.info("BookKeeper Client created {} with shared zookeeper client : ledgersPath = {}, numRetries = {}, " + "sessionTimeout = {}, backoff = {}, maxBackoff = {}, dnsResolver = {}, registerExpirationHandler = {}", new Object[] { name, ledgersPath, conf.getZKNumRetries(), conf.getZKSessionTimeoutMilliseconds(), conf.getZKRetryBackoffStartMillis(), conf.getZKRetryBackoffMaxMillis(), conf.getBkDNSResolverOverrides(), registerExpirationHandler });
}
}
Aggregations