Examples with BKDLConfig com.twitter.distributedlog.metadata.BKDLConfig used on opensource projects

Example 6 with BKDLConfig

use of com.twitter.distributedlog.metadata.BKDLConfig in project distributedlog by twitter.

the class LedgerReadBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    List<LogSegmentMetadata> segments = null;
    while (null == segments) {
        try {
            segments = dlm.getLogSegments();
        } catch (IOException ioe) {
            logger.warn("Failed to get log segments for stream {} : ", streamName, ioe);
        }
        if (null == segments) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    final Counter readCounter = statsLogger.getCounter("reads");
    logger.info("Reading from log segments : {}", segments);
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri).name("benchmark-zkc").sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).zkAclId(null).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException e) {
        return;
    }
    BookKeeper bk;
    try {
        bk = BookKeeperClientBuilder.newBuilder().name("benchmark-bkc").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForReader()).ledgersPath(bkdlConfig.getBkLedgersPath()).build().get();
    } catch (IOException e) {
        return;
    }
    final int readConcurrency = conf.getInt("ledger_read_concurrency", 1000);
    boolean streamRead = conf.getBoolean("ledger_stream_read", true);
    try {
        for (LogSegmentMetadata segment : segments) {
            Stopwatch stopwatch = Stopwatch.createStarted();
            long lid = segment.getLedgerId();
            LedgerHandle lh = bk.openLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
            logger.info("It took {} ms to open log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
            stopwatch.reset().start();
            Runnable reader;
            if (streamRead) {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            } else {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            }
            reader.run();
            logger.info("It took {} ms to complete reading {} entries from log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
        }
    } catch (Exception e) {
        logger.error("Error on reading bk ", e);
    }
}

Example 7 with BKDLConfig

use of com.twitter.distributedlog.metadata.BKDLConfig in project distributedlog by twitter.

the class DLMTestUtil method createNewBKDLM.

static BKLogPartitionWriteHandlerAndClients createNewBKDLM(DistributedLogConfiguration conf, String logName, int zkPort) throws Exception {
    URI uri = createDLMURI(zkPort, "/" + logName);
    ZooKeeperClientBuilder zkcBuilder = TestZooKeeperClientBuilder.newBuilder(conf).name(String.format("dlzk:%s:handler_dedicated", logName)).uri(uri);
    ZooKeeperClient zkClient = zkcBuilder.build();
    try {
        zkClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (KeeperException.NodeExistsException nee) {
    // ignore
    }
    // resolve uri
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkClient, uri);
    BKDLConfig.propagateConfiguration(bkdlConfig, conf);
    BookKeeperClientBuilder bkcBuilder = BookKeeperClientBuilder.newBuilder().dlConfig(conf).name(String.format("bk:%s:handler_dedicated", logName)).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).statsLogger(NullStatsLogger.INSTANCE);
    BKDistributedLogManager bkdlm = new BKDistributedLogManager(logName, conf, uri, zkcBuilder, zkcBuilder, zkClient, zkClient, bkcBuilder, bkcBuilder, new SettableFeatureProvider("", 0), PermitLimiter.NULL_PERMIT_LIMITER, NullStatsLogger.INSTANCE);
    BKLogWriteHandler writeHandler = bkdlm.createWriteHandler(true);
    return new BKLogPartitionWriteHandlerAndClients(writeHandler, zkClient, bkcBuilder.build());
}

Example 8 with BKDLConfig

use of com.twitter.distributedlog.metadata.BKDLConfig in project distributedlog by twitter.

the class DLMTestUtil method updateBKDLConfig.

static void updateBKDLConfig(URI uri, String zkServers, String ledgersPath, boolean sanityCheckTxnID) throws Exception {
    BKDLConfig bkdlConfig = new BKDLConfig(zkServers, ledgersPath).setSanityCheckTxnID(sanityCheckTxnID);
    DLMetadata.create(bkdlConfig).update(uri);
}

Example 9 with BKDLConfig

use of com.twitter.distributedlog.metadata.BKDLConfig in project distributedlog by twitter.

the class TestBKLogSegmentWriter method setup.

@Before
@Override
public void setup() throws Exception {
    super.setup();
    scheduler = OrderedScheduler.newBuilder().corePoolSize(1).build();
    lockStateExecutor = OrderedScheduler.newBuilder().corePoolSize(1).build();
    // build zookeeper client
    URI uri = createDLMURI("");
    zkc = TestZooKeeperClientBuilder.newBuilder(conf).name("test-zkc").uri(uri).build();
    zkc0 = TestZooKeeperClientBuilder.newBuilder(conf).name("test-zkc0").uri(uri).build();
    // build bookkeeper client
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    bkc = BookKeeperClientBuilder.newBuilder().dlConfig(conf).name("test-bkc").ledgersPath(bkdlConfig.getBkLedgersPath()).zkServers(DLUtils.getZKServersFromDLUri(uri)).build();
}

Example 10 with BKDLConfig

use of com.twitter.distributedlog.metadata.BKDLConfig in project distributedlog by twitter.

the class DLAuditor method resolveBKDLConfig.

private BKDLConfig resolveBKDLConfig(ZooKeeperClient zkc, List<URI> uris) throws IOException {
    URI firstURI = uris.get(0);
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, firstURI);
    for (URI uri : uris) {
        BKDLConfig anotherConfig = BKDLConfig.resolveDLConfig(zkc, uri);
        if (!(Objects.equal(bkdlConfig.getBkLedgersPath(), anotherConfig.getBkLedgersPath()) && Objects.equal(bkdlConfig.getBkZkServersForWriter(), anotherConfig.getBkZkServersForWriter()))) {
            throw new IllegalArgumentException("Uris don't use same bookkeeper cluster");
        }
    }
    return bkdlConfig;
}