Examples with OpStatsLogger org.apache.bookkeeper.stats.OpStatsLogger used on opensource projects

Example 1 with OpStatsLogger

use of org.apache.bookkeeper.stats.OpStatsLogger in project distributedlog by twitter.

the class AsyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    // Stats
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter readCounter = statsLogger.getCounter("reads");
    AsyncLogReader reader = null;
    DLSN lastDLSN = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    lastDLSN = DLSN.InitialDLSN;
                    break;
                case LATEST:
                    lastTxId = Long.MAX_VALUE;
                    try {
                        lastDLSN = dlm.getLastDLSN();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    lastDLSN = null;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    lastDLSN = null;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            if (null == lastDLSN) {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastTxId));
            } else {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastDLSN));
            }
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from tx id = {}, dlsn = {}.", new Object[] { streamName, lastTxId, lastDLSN });
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        List<LogRecordWithDLSN> records;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                records = FutureUtils.result(reader.readBulk(batchSize));
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                if (!records.isEmpty()) {
                    readCounter.add(records.size());
                    LogRecordWithDLSN lastRecord = records.get(records.size() - 1);
                    lastTxId = lastRecord.getTransactionId();
                    lastDLSN = lastRecord.getDlsn();
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 2 with OpStatsLogger

use of org.apache.bookkeeper.stats.OpStatsLogger in project distributedlog by twitter.

the class BKLogHandler method asyncGetLedgerList.

protected Future<List<LogSegmentMetadata>> asyncGetLedgerList(final boolean forceFetch, final boolean fetchFullList, final Comparator<LogSegmentMetadata> comparator, final boolean throwOnEmpty) {
    final Promise<List<LogSegmentMetadata>> promise = new Promise<List<LogSegmentMetadata>>();
    final Stopwatch stopwatch = Stopwatch.createStarted();
    final OpStatsLogger statsLogger = fetchFullList ? getFullListStat : getFilteredListStat;
    asyncDoGetLedgerList(forceFetch, fetchFullList, comparator, throwOnEmpty).addEventListener(new FutureEventListener<List<LogSegmentMetadata>>() {

        @Override
        public void onSuccess(List<LogSegmentMetadata> value) {
            statsLogger.registerSuccessfulEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
            promise.setValue(value);
        }

        @Override
        public void onFailure(Throwable cause) {
            statsLogger.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
            promise.setException(cause);
        }
    });
    return promise;
}

Example 3 with OpStatsLogger

use of org.apache.bookkeeper.stats.OpStatsLogger in project distributedlog by twitter.

the class BKLogHandler method getLedgerList.

protected List<LogSegmentMetadata> getLedgerList(boolean forceFetch, boolean fetchFullList, Comparator<LogSegmentMetadata> comparator, boolean throwOnEmpty) throws IOException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    boolean success = false;
    try {
        List<LogSegmentMetadata> segments = doGetLedgerList(forceFetch, fetchFullList, comparator, throwOnEmpty);
        success = true;
        return segments;
    } finally {
        OpStatsLogger statsLogger = fetchFullList ? getFullListStat : getFilteredListStat;
        if (success) {
            statsLogger.registerSuccessfulEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        } else {
            statsLogger.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        }
    }
}

Example 4 with OpStatsLogger

use of org.apache.bookkeeper.stats.OpStatsLogger in project distributedlog by twitter.

the class SyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String streamName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger nonBlockingReadStats = statsLogger.getOpStatsLogger("non_blocking_read");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter nullReadCounter = statsLogger.getCounter("null_read");
    logger.info("Created dlm for stream {}.", streamName);
    LogReader reader = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    break;
                case LATEST:
                    try {
                        lastTxId = dlm.getLastTxId();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id {}", lastTxId);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            reader = dlm.getInputStream(lastTxId);
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id {}", lastTxId);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from {}.", streamName, lastTxId);
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        // read loop
        LogRecord record;
        boolean nonBlocking = false;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                record = reader.readNext(nonBlocking);
                if (null != record) {
                    long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                    if (nonBlocking) {
                        nonBlockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    } else {
                        blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    }
                    lastTxId = record.getTransactionId();
                } else {
                    nullReadCounter.inc();
                }
                if (null == record && !nonBlocking) {
                    nonBlocking = true;
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 5 with OpStatsLogger

use of org.apache.bookkeeper.stats.OpStatsLogger in project distributedlog by twitter.

the class LedgerHandleCache method asyncOpenLedger.

/**
     * Open the log segment.
     *
     * @param metadata
     *          the log segment metadata
     * @param fence
     *          whether to fence the log segment during open
     * @return a future presenting the open result.
     */
public Future<LedgerDescriptor> asyncOpenLedger(LogSegmentMetadata metadata, boolean fence) {
    final Stopwatch stopwatch = Stopwatch.createStarted();
    final OpStatsLogger openStatsLogger = fence ? openStats : openNoRecoveryStats;
    final Promise<LedgerDescriptor> promise = new Promise<LedgerDescriptor>();
    final LedgerDescriptor ledgerDesc = new LedgerDescriptor(metadata.getLedgerId(), metadata.getLogSegmentSequenceNumber(), fence);
    RefCountedLedgerHandle refhandle = handlesMap.get(ledgerDesc);
    if (null == refhandle) {
        asyncOpenLedger(ledgerDesc, new AsyncCallback.OpenCallback() {

            @Override
            public void openComplete(int rc, LedgerHandle lh, Object ctx) {
                if (BKException.Code.OK != rc) {
                    promise.setException(BKException.create(rc));
                    return;
                }
                RefCountedLedgerHandle newRefHandle = new RefCountedLedgerHandle(lh);
                RefCountedLedgerHandle oldRefHandle = handlesMap.putIfAbsent(ledgerDesc, newRefHandle);
                if (null != oldRefHandle) {
                    oldRefHandle.addRef();
                    if (newRefHandle.removeRef()) {
                        newRefHandle.handle.asyncClose(new AsyncCallback.CloseCallback() {

                            @Override
                            public void closeComplete(int i, LedgerHandle ledgerHandle, Object o) {
                            // No action necessary
                            }
                        }, null);
                    }
                }
                promise.setValue(ledgerDesc);
            }
        }, null);
    } else {
        refhandle.addRef();
        promise.setValue(ledgerDesc);
    }
    return promise.addEventListener(new FutureEventListener<LedgerDescriptor>() {

        @Override
        public void onSuccess(LedgerDescriptor value) {
            openStatsLogger.registerSuccessfulEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
        }

        @Override
        public void onFailure(Throwable cause) {
            openStatsLogger.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
        }
    });
}