Examples with Stopwatch com.google.common.base.Stopwatch used on opensource projects

Example 31 with Stopwatch

use of com.google.common.base.Stopwatch in project hbase by apache.

the class ScanPerformanceEvaluation method testHdfsStreaming.

protected void testHdfsStreaming(Path filename) throws IOException {
    byte[] buf = new byte[1024];
    FileSystem fs = filename.getFileSystem(getConf());
    // read the file from start to finish
    Stopwatch fileOpenTimer = new Stopwatch();
    Stopwatch streamTimer = new Stopwatch();
    fileOpenTimer.start();
    FSDataInputStream in = fs.open(filename);
    fileOpenTimer.stop();
    long totalBytes = 0;
    streamTimer.start();
    while (true) {
        int read = in.read(buf);
        if (read < 0) {
            break;
        }
        totalBytes += read;
    }
    streamTimer.stop();
    double throughput = (double) totalBytes / streamTimer.elapsedTime(TimeUnit.SECONDS);
    System.out.println("HDFS streaming: ");
    System.out.println("total time to open: " + fileOpenTimer.elapsedMillis() + " ms");
    System.out.println("total time to read: " + streamTimer.elapsedMillis() + " ms");
    System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")");
    System.out.println("throghput  : " + StringUtils.humanReadableInt((long) throughput) + "B/s");
}

Example 32 with Stopwatch

use of com.google.common.base.Stopwatch in project buck by facebook.

the class ThriftCoordinatorClient method start.

public ThriftCoordinatorClient start() throws IOException {
    transport = new TFramedTransport(new TSocket(remoteHost, remotePort));
    Stopwatch stopwatch = Stopwatch.createStarted();
    while (true) {
        try {
            transport.open();
            break;
        } catch (TTransportException e) {
            if (stopwatch.elapsed(TimeUnit.SECONDS) > MAX_CONNECT_TIMEOUT_SECONDS) {
                throw new IOException(String.format("Failed to connect. Coordinator is still not healthy after [%d] seconds.", MAX_CONNECT_TIMEOUT_SECONDS));
            }
            LOG.debug("Coordinator server currently not available. Retrying in a bit...");
            try {
                Thread.sleep(TimeUnit.SECONDS.toMillis(RETRY_TIMEOUT_SECONDS));
            } catch (InterruptedException innerException) {
                throw new RuntimeException(innerException);
            }
        }
    }
    TProtocol protocol = new TBinaryProtocol(transport);
    client = new CoordinatorService.Client(protocol);
    return this;
}

Example 33 with Stopwatch

use of com.google.common.base.Stopwatch in project buck by facebook.

the class DistBuildClientExecutor method executeAndPrintFailuresToEventBus.

public int executeAndPrintFailuresToEventBus(final WeightedListeningExecutorService executorService, ProjectFilesystem projectFilesystem, FileHashCache fileHashCache, BuckEventBus eventBus) throws IOException, InterruptedException {
    BuildJob job = distBuildService.createBuild();
    final StampedeId id = job.getStampedeId();
    LOG.info("Created job. Build id = " + id.getId());
    logDebugInfo(job);
    List<ListenableFuture<Void>> asyncJobs = new LinkedList<>();
    LOG.info("Uploading local changes.");
    asyncJobs.add(distBuildService.uploadMissingFiles(buildJobState.fileHashes, executorService));
    LOG.info("Uploading target graph.");
    asyncJobs.add(distBuildService.uploadTargetGraph(buildJobState, id, executorService));
    LOG.info("Uploading buck dot-files.");
    asyncJobs.add(distBuildService.uploadBuckDotFiles(id, projectFilesystem, fileHashCache, executorService));
    try {
        Futures.allAsList(asyncJobs).get();
    } catch (ExecutionException e) {
        LOG.error("Upload failed.");
        throw new RuntimeException(e);
    }
    distBuildService.setBuckVersion(id, buckVersion);
    LOG.info("Set Buck Version. Build status: " + job.getStatus().toString());
    job = distBuildService.startBuild(id);
    LOG.info("Started job. Build status: " + job.getStatus().toString());
    logDebugInfo(job);
    Stopwatch stopwatch = Stopwatch.createStarted();
    // Keep polling until the build is complete or failed.
    do {
        job = distBuildService.getCurrentBuildJobState(id);
        LOG.info("Got build status: " + job.getStatus().toString());
        DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(MAX_BUILD_DURATION_MILLIS - stopwatch.elapsed(TimeUnit.MILLISECONDS)).build();
        eventBus.post(new DistBuildStatusEvent(distBuildStatus));
        List<LogLineBatchRequest> newLogLineRequests = distBuildLogStateTracker.createRealtimeLogRequests(job.getSlaveInfoByRunId().values());
        MultiGetBuildSlaveRealTimeLogsResponse slaveLogsResponse = distBuildService.fetchSlaveLogLines(job.stampedeId, newLogLineRequests);
        distBuildLogStateTracker.processStreamLogs(slaveLogsResponse.getMultiStreamLogs());
        try {
            // TODO(shivanker): Get rid of sleeps in methods which we want to unit test
            Thread.sleep(millisBetweenStatusPoll);
        } catch (InterruptedException e) {
            LOG.error(e, "BuildStatus polling sleep call has been interrupted unexpectedly.");
        }
    } while (!(job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) || job.getStatus().equals(BuildStatus.FAILED)));
    try {
        MultiGetBuildSlaveLogDirResponse logDirsResponse = distBuildService.fetchBuildSlaveLogDir(job.stampedeId, distBuildLogStateTracker.runIdsToMaterializeLogDirsFor(job.slaveInfoByRunId.values()));
        distBuildLogStateTracker.materializeLogDirs(logDirsResponse.getLogDirs());
    } catch (IOException ex) {
        LOG.error(ex);
    }
    LOG.info("Build was " + (job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? "" : "not ") + "successful!");
    logDebugInfo(job);
    DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(0).build();
    eventBus.post(new DistBuildStatusEvent(distBuildStatus));
    return job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? 0 : 1;
}

Example 34 with Stopwatch

use of com.google.common.base.Stopwatch in project hbase by apache.

the class ScanPerformanceEvaluation method testScan.

public void testScan() throws IOException {
    Stopwatch tableOpenTimer = new Stopwatch();
    Stopwatch scanOpenTimer = new Stopwatch();
    Stopwatch scanTimer = new Stopwatch();
    tableOpenTimer.start();
    Connection connection = ConnectionFactory.createConnection(getConf());
    Table table = connection.getTable(TableName.valueOf(tablename));
    tableOpenTimer.stop();
    Scan scan = getScan();
    scanOpenTimer.start();
    ResultScanner scanner = table.getScanner(scan);
    scanOpenTimer.stop();
    long numRows = 0;
    long numCells = 0;
    scanTimer.start();
    while (true) {
        Result result = scanner.next();
        if (result == null) {
            break;
        }
        numRows++;
        numCells += result.rawCells().length;
    }
    scanTimer.stop();
    scanner.close();
    table.close();
    connection.close();
    ScanMetrics metrics = scan.getScanMetrics();
    long totalBytes = metrics.countOfBytesInResults.get();
    double throughput = (double) totalBytes / scanTimer.elapsedTime(TimeUnit.SECONDS);
    double throughputRows = (double) numRows / scanTimer.elapsedTime(TimeUnit.SECONDS);
    double throughputCells = (double) numCells / scanTimer.elapsedTime(TimeUnit.SECONDS);
    System.out.println("HBase scan: ");
    System.out.println("total time to open table: " + tableOpenTimer.elapsedMillis() + " ms");
    System.out.println("total time to open scanner: " + scanOpenTimer.elapsedMillis() + " ms");
    System.out.println("total time to scan: " + scanTimer.elapsedMillis() + " ms");
    System.out.println("Scan metrics:\n" + metrics.getMetricsMap());
    System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")");
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughput) + "B/s");
    System.out.println("total rows  : " + numRows);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputRows) + " rows/s");
    System.out.println("total cells : " + numCells);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputCells) + " cells/s");
}

Example 35 with Stopwatch

use of com.google.common.base.Stopwatch in project hbase by apache.

the class ScanPerformanceEvaluation method testSnapshotScan.

public void testSnapshotScan() throws IOException {
    Stopwatch snapshotRestoreTimer = new Stopwatch();
    Stopwatch scanOpenTimer = new Stopwatch();
    Stopwatch scanTimer = new Stopwatch();
    Path restoreDir = new Path(this.restoreDir);
    snapshotRestoreTimer.start();
    restoreDir.getFileSystem(conf).delete(restoreDir, true);
    snapshotRestoreTimer.stop();
    Scan scan = getScan();
    scanOpenTimer.start();
    TableSnapshotScanner scanner = new TableSnapshotScanner(conf, restoreDir, snapshotName, scan);
    scanOpenTimer.stop();
    long numRows = 0;
    long numCells = 0;
    scanTimer.start();
    while (true) {
        Result result = scanner.next();
        if (result == null) {
            break;
        }
        numRows++;
        numCells += result.rawCells().length;
    }
    scanTimer.stop();
    scanner.close();
    ScanMetrics metrics = scanner.getScanMetrics();
    long totalBytes = metrics.countOfBytesInResults.get();
    double throughput = (double) totalBytes / scanTimer.elapsedTime(TimeUnit.SECONDS);
    double throughputRows = (double) numRows / scanTimer.elapsedTime(TimeUnit.SECONDS);
    double throughputCells = (double) numCells / scanTimer.elapsedTime(TimeUnit.SECONDS);
    System.out.println("HBase scan snapshot: ");
    System.out.println("total time to restore snapshot: " + snapshotRestoreTimer.elapsedMillis() + " ms");
    System.out.println("total time to open scanner: " + scanOpenTimer.elapsedMillis() + " ms");
    System.out.println("total time to scan: " + scanTimer.elapsedMillis() + " ms");
    System.out.println("Scan metrics:\n" + metrics.getMetricsMap());
    System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")");
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughput) + "B/s");
    System.out.println("total rows  : " + numRows);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputRows) + " rows/s");
    System.out.println("total cells : " + numCells);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputCells) + " cells/s");
}