Examples with ZooKeeperClient com.twitter.distributedlog.ZooKeeperClient used on opensource projects

Example 11 with ZooKeeperClient

use of com.twitter.distributedlog.ZooKeeperClient in project distributedlog by twitter.

the class LedgerReadBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    List<LogSegmentMetadata> segments = null;
    while (null == segments) {
        try {
            segments = dlm.getLogSegments();
        } catch (IOException ioe) {
            logger.warn("Failed to get log segments for stream {} : ", streamName, ioe);
        }
        if (null == segments) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    final Counter readCounter = statsLogger.getCounter("reads");
    logger.info("Reading from log segments : {}", segments);
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri).name("benchmark-zkc").sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).zkAclId(null).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException e) {
        return;
    }
    BookKeeper bk;
    try {
        bk = BookKeeperClientBuilder.newBuilder().name("benchmark-bkc").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForReader()).ledgersPath(bkdlConfig.getBkLedgersPath()).build().get();
    } catch (IOException e) {
        return;
    }
    final int readConcurrency = conf.getInt("ledger_read_concurrency", 1000);
    boolean streamRead = conf.getBoolean("ledger_stream_read", true);
    try {
        for (LogSegmentMetadata segment : segments) {
            Stopwatch stopwatch = Stopwatch.createStarted();
            long lid = segment.getLedgerId();
            LedgerHandle lh = bk.openLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
            logger.info("It took {} ms to open log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
            stopwatch.reset().start();
            Runnable reader;
            if (streamRead) {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            } else {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            }
            reader.run();
            logger.info("It took {} ms to complete reading {} entries from log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
        }
    } catch (Exception e) {
        logger.error("Error on reading bk ", e);
    }
}

Example 12 with ZooKeeperClient

use of com.twitter.distributedlog.ZooKeeperClient in project distributedlog by twitter.

the class TestFederatedZKLogMetadataStore method testZooKeeperSessionExpired.

@Test(timeout = 60000)
public void testZooKeeperSessionExpired() throws Exception {
    Set<String> allLogs = createLogs(2 * maxLogsPerSubnamespace, "test-zookeeper-session-expired-");
    TestNamespaceListenerWithExpectedSize listener = new TestNamespaceListenerWithExpectedSize(2 * maxLogsPerSubnamespace + 1);
    metadataStore.registerNamespaceListener(listener);
    ZooKeeperClientUtils.expireSession(zkc, DLUtils.getZKServersFromDLUri(uri), zkSessionTimeoutMs);
    String testLogName = "test-log-name";
    allLogs.add(testLogName);
    DistributedLogConfiguration anotherConf = new DistributedLogConfiguration();
    anotherConf.addConfiguration(baseConf);
    ZooKeeperClient anotherZkc = TestZooKeeperClientBuilder.newBuilder().uri(uri).sessionTimeoutMs(zkSessionTimeoutMs).build();
    FederatedZKLogMetadataStore anotherMetadataStore = new FederatedZKLogMetadataStore(anotherConf, uri, anotherZkc, scheduler);
    FutureUtils.result(anotherMetadataStore.createLog(testLogName));
    listener.waitForDone();
    Set<String> receivedLogs = listener.getResult();
    assertEquals(2 * maxLogsPerSubnamespace + 1, receivedLogs.size());
    assertEquals(allLogs, receivedLogs);
}