Examples with Counter org.apache.bookkeeper.stats.Counter used on opensource projects

Example 1 with Counter

use of org.apache.bookkeeper.stats.Counter in project distributedlog by twitter.

the class AsyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    // Stats
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter readCounter = statsLogger.getCounter("reads");
    AsyncLogReader reader = null;
    DLSN lastDLSN = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    lastDLSN = DLSN.InitialDLSN;
                    break;
                case LATEST:
                    lastTxId = Long.MAX_VALUE;
                    try {
                        lastDLSN = dlm.getLastDLSN();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    lastDLSN = null;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    lastDLSN = null;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            if (null == lastDLSN) {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastTxId));
            } else {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastDLSN));
            }
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from tx id = {}, dlsn = {}.", new Object[] { streamName, lastTxId, lastDLSN });
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        List<LogRecordWithDLSN> records;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                records = FutureUtils.result(reader.readBulk(batchSize));
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                if (!records.isEmpty()) {
                    readCounter.add(records.size());
                    LogRecordWithDLSN lastRecord = records.get(records.size() - 1);
                    lastTxId = lastRecord.getTransactionId();
                    lastDLSN = lastRecord.getDlsn();
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 2 with Counter

use of org.apache.bookkeeper.stats.Counter in project distributedlog by twitter.

the class DistributedLogServiceImpl method countStatusCode.

private void countStatusCode(StatusCode code) {
    Counter counter = statusCodeCounters.get(code);
    if (null == counter) {
        counter = statusCodeStatLogger.getCounter(code.name());
        Counter oldCounter = statusCodeCounters.putIfAbsent(code, counter);
        if (null != oldCounter) {
            counter = oldCounter;
        }
    }
    counter.inc();
    statusCodeTotal.inc();
}

Example 3 with Counter

use of org.apache.bookkeeper.stats.Counter in project distributedlog by twitter.

the class StreamImpl method countException.

void countException(Throwable t, StatsLogger streamExceptionLogger) {
    String exceptionName = null == t ? "null" : t.getClass().getName();
    Counter counter = exceptionCounters.get(exceptionName);
    if (null == counter) {
        counter = exceptionStatLogger.getCounter(exceptionName);
        Counter oldCounter = exceptionCounters.putIfAbsent(exceptionName, counter);
        if (null != oldCounter) {
            counter = oldCounter;
        }
    }
    counter.inc();
    streamExceptionLogger.getCounter(exceptionName).inc();
}

Example 4 with Counter

use of org.apache.bookkeeper.stats.Counter in project distributedlog by twitter.

the class LedgerReadBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    List<LogSegmentMetadata> segments = null;
    while (null == segments) {
        try {
            segments = dlm.getLogSegments();
        } catch (IOException ioe) {
            logger.warn("Failed to get log segments for stream {} : ", streamName, ioe);
        }
        if (null == segments) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    final Counter readCounter = statsLogger.getCounter("reads");
    logger.info("Reading from log segments : {}", segments);
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri).name("benchmark-zkc").sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).zkAclId(null).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException e) {
        return;
    }
    BookKeeper bk;
    try {
        bk = BookKeeperClientBuilder.newBuilder().name("benchmark-bkc").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForReader()).ledgersPath(bkdlConfig.getBkLedgersPath()).build().get();
    } catch (IOException e) {
        return;
    }
    final int readConcurrency = conf.getInt("ledger_read_concurrency", 1000);
    boolean streamRead = conf.getBoolean("ledger_stream_read", true);
    try {
        for (LogSegmentMetadata segment : segments) {
            Stopwatch stopwatch = Stopwatch.createStarted();
            long lid = segment.getLedgerId();
            LedgerHandle lh = bk.openLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
            logger.info("It took {} ms to open log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
            stopwatch.reset().start();
            Runnable reader;
            if (streamRead) {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            } else {
                reader = new LedgerStreamReader(lh, new BookkeeperInternalCallbacks.ReadEntryListener() {

                    @Override
                    public void onEntryComplete(int rc, LedgerHandle lh, LedgerEntry entry, Object ctx) {
                        readCounter.inc();
                    }
                }, readConcurrency);
            }
            reader.run();
            logger.info("It took {} ms to complete reading {} entries from log segment {}", new Object[] { stopwatch.elapsed(TimeUnit.MILLISECONDS), (lh.getLastAddConfirmed() + 1), segment });
        }
    } catch (Exception e) {
        logger.error("Error on reading bk ", e);
    }
}

Example 5 with Counter

use of org.apache.bookkeeper.stats.Counter in project distributedlog by twitter.

the class SyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String streamName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger nonBlockingReadStats = statsLogger.getOpStatsLogger("non_blocking_read");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter nullReadCounter = statsLogger.getCounter("null_read");
    logger.info("Created dlm for stream {}.", streamName);
    LogReader reader = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    break;
                case LATEST:
                    try {
                        lastTxId = dlm.getLastTxId();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id {}", lastTxId);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            reader = dlm.getInputStream(lastTxId);
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id {}", lastTxId);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from {}.", streamName, lastTxId);
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        // read loop
        LogRecord record;
        boolean nonBlocking = false;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                record = reader.readNext(nonBlocking);
                if (null != record) {
                    long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                    if (nonBlocking) {
                        nonBlockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    } else {
                        blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    }
                    lastTxId = record.getTransactionId();
                } else {
                    nullReadCounter.inc();
                }
                if (null == record && !nonBlocking) {
                    nonBlocking = true;
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}