Examples with ScanMetrics org.apache.hadoop.hbase.client.metrics.ScanMetrics used on opensource projects

Example 1 with ScanMetrics

use of org.apache.hadoop.hbase.client.metrics.ScanMetrics in project hbase by apache.

the class TestFromClientSide method testScanMetrics.

/**
  * Test ScanMetrics
  * @throws Exception
  */
@Test
@SuppressWarnings("unused")
public void testScanMetrics() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    // Set up test table:
    // Create table:
    Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY);
    int numOfRegions = -1;
    try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
        numOfRegions = r.getStartKeys().length;
    }
    // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
    // scan are forced to hit all the regions.
    Put put1 = new Put(Bytes.toBytes("zzz1"));
    put1.addColumn(FAMILY, QUALIFIER, VALUE);
    Put put2 = new Put(Bytes.toBytes("zzz2"));
    put2.addColumn(FAMILY, QUALIFIER, VALUE);
    Put put3 = new Put(Bytes.toBytes("zzz3"));
    put3.addColumn(FAMILY, QUALIFIER, VALUE);
    ht.put(Arrays.asList(put1, put2, put3));
    Scan scan1 = new Scan();
    int numRecords = 0;
    ResultScanner scanner = ht.getScanner(scan1);
    for (Result result : scanner) {
        numRecords++;
    }
    scanner.close();
    LOG.info("test data has " + numRecords + " records.");
    // by default, scan metrics collection is turned off
    assertEquals(null, scan1.getScanMetrics());
    // turn on scan metrics
    Scan scan2 = new Scan();
    scan2.setScanMetricsEnabled(true);
    scan2.setCaching(numRecords + 1);
    scanner = ht.getScanner(scan2);
    for (Result result : scanner.next(numRecords - 1)) {
    }
    scanner.close();
    // closing the scanner will set the metrics.
    assertNotNull(scan2.getScanMetrics());
    // set caching to 1, because metrics are collected in each roundtrip only
    scan2 = new Scan();
    scan2.setScanMetricsEnabled(true);
    scan2.setCaching(1);
    scanner = ht.getScanner(scan2);
    // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
    for (Result result : scanner.next(numRecords - 1)) {
    }
    scanner.close();
    ScanMetrics scanMetrics = scan2.getScanMetrics();
    assertEquals("Did not access all the regions in the table", numOfRegions, scanMetrics.countOfRegions.get());
    // check byte counters
    scan2 = new Scan();
    scan2.setScanMetricsEnabled(true);
    scan2.setCaching(1);
    scanner = ht.getScanner(scan2);
    int numBytes = 0;
    for (Result result : scanner.next(1)) {
        for (Cell cell : result.listCells()) {
            numBytes += CellUtil.estimatedSerializedSizeOf(cell);
        }
    }
    scanner.close();
    scanMetrics = scan2.getScanMetrics();
    assertEquals("Did not count the result bytes", numBytes, scanMetrics.countOfBytesInResults.get());
    // check byte counters on a small scan
    scan2 = new Scan();
    scan2.setScanMetricsEnabled(true);
    scan2.setCaching(1);
    scan2.setSmall(true);
    scanner = ht.getScanner(scan2);
    numBytes = 0;
    for (Result result : scanner.next(1)) {
        for (Cell cell : result.listCells()) {
            numBytes += CellUtil.estimatedSerializedSizeOf(cell);
        }
    }
    scanner.close();
    scanMetrics = scan2.getScanMetrics();
    assertEquals("Did not count the result bytes", numBytes, scanMetrics.countOfBytesInResults.get());
    // now, test that the metrics are still collected even if you don't call close, but do
    // run past the end of all the records
    /** There seems to be a timing issue here.  Comment out for now. Fix when time.
    Scan scanWithoutClose = new Scan();
    scanWithoutClose.setCaching(1);
    scanWithoutClose.setScanMetricsEnabled(true);
    ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
    for (Result result : scannerWithoutClose.next(numRecords + 1)) {
    }
    ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
    assertEquals("Did not access all the regions in the table", numOfRegions,
        scanMetricsWithoutClose.countOfRegions.get());
    */
    // finally, test that the metrics are collected correctly if you both run past all the records,
    // AND close the scanner
    Scan scanWithClose = new Scan();
    // make sure we can set caching up to the number of a scanned values
    scanWithClose.setCaching(numRecords);
    scanWithClose.setScanMetricsEnabled(true);
    ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
    for (Result result : scannerWithClose.next(numRecords + 1)) {
    }
    scannerWithClose.close();
    ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
    assertEquals("Did not access all the regions in the table", numOfRegions, scanMetricsWithClose.countOfRegions.get());
}

Example 2 with ScanMetrics

use of org.apache.hadoop.hbase.client.metrics.ScanMetrics in project hbase by apache.

the class ProtobufUtil method toScanMetrics.

public static ScanMetrics toScanMetrics(final byte[] bytes) {
    Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
    MapReduceProtos.ScanMetrics pScanMetrics = null;
    try {
        pScanMetrics = parser.parseFrom(bytes);
    } catch (InvalidProtocolBufferException e) {
    //Ignored there are just no key values to add.
    }
    ScanMetrics scanMetrics = new ScanMetrics();
    if (pScanMetrics != null) {
        for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
            if (pair.hasName() && pair.hasValue()) {
                scanMetrics.setCounter(pair.getName(), pair.getValue());
            }
        }
    }
    return scanMetrics;
}

Example 3 with ScanMetrics

use of org.apache.hadoop.hbase.client.metrics.ScanMetrics in project hbase by apache.

the class ScanPerformanceEvaluation method testSnapshotScan.

public void testSnapshotScan() throws IOException {
    Stopwatch snapshotRestoreTimer = Stopwatch.createUnstarted();
    Stopwatch scanOpenTimer = Stopwatch.createUnstarted();
    Stopwatch scanTimer = Stopwatch.createUnstarted();
    Path restoreDir = new Path(this.restoreDir);
    snapshotRestoreTimer.start();
    restoreDir.getFileSystem(conf).delete(restoreDir, true);
    snapshotRestoreTimer.stop();
    Scan scan = getScan();
    scanOpenTimer.start();
    TableSnapshotScanner scanner = new TableSnapshotScanner(conf, restoreDir, snapshotName, scan);
    scanOpenTimer.stop();
    long numRows = 0;
    long numCells = 0;
    scanTimer.start();
    while (true) {
        Result result = scanner.next();
        if (result == null) {
            break;
        }
        numRows++;
        numCells += result.rawCells().length;
    }
    scanTimer.stop();
    scanner.close();
    ScanMetrics metrics = scanner.getScanMetrics();
    long totalBytes = metrics.countOfBytesInResults.get();
    double throughput = (double) totalBytes / scanTimer.elapsed(TimeUnit.SECONDS);
    double throughputRows = (double) numRows / scanTimer.elapsed(TimeUnit.SECONDS);
    double throughputCells = (double) numCells / scanTimer.elapsed(TimeUnit.SECONDS);
    System.out.println("HBase scan snapshot: ");
    System.out.println("total time to restore snapshot: " + snapshotRestoreTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("total time to open scanner: " + scanOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("total time to scan: " + scanTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("Scan metrics:\n" + metrics.getMetricsMap());
    System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")");
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughput) + "B/s");
    System.out.println("total rows  : " + numRows);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputRows) + " rows/s");
    System.out.println("total cells : " + numCells);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputCells) + " cells/s");
}

Example 4 with ScanMetrics

use of org.apache.hadoop.hbase.client.metrics.ScanMetrics in project hbase by apache.

the class ScanPerformanceEvaluation method testScan.

public void testScan() throws IOException {
    Stopwatch tableOpenTimer = Stopwatch.createUnstarted();
    Stopwatch scanOpenTimer = Stopwatch.createUnstarted();
    Stopwatch scanTimer = Stopwatch.createUnstarted();
    tableOpenTimer.start();
    Connection connection = ConnectionFactory.createConnection(getConf());
    Table table = connection.getTable(TableName.valueOf(tablename));
    tableOpenTimer.stop();
    Scan scan = getScan();
    scanOpenTimer.start();
    ResultScanner scanner = table.getScanner(scan);
    scanOpenTimer.stop();
    long numRows = 0;
    long numCells = 0;
    scanTimer.start();
    while (true) {
        Result result = scanner.next();
        if (result == null) {
            break;
        }
        numRows++;
        numCells += result.rawCells().length;
    }
    scanTimer.stop();
    scanner.close();
    table.close();
    connection.close();
    ScanMetrics metrics = scanner.getScanMetrics();
    long totalBytes = metrics.countOfBytesInResults.get();
    double throughput = (double) totalBytes / scanTimer.elapsed(TimeUnit.SECONDS);
    double throughputRows = (double) numRows / scanTimer.elapsed(TimeUnit.SECONDS);
    double throughputCells = (double) numCells / scanTimer.elapsed(TimeUnit.SECONDS);
    System.out.println("HBase scan: ");
    System.out.println("total time to open table: " + tableOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("total time to open scanner: " + scanOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("total time to scan: " + scanTimer.elapsed(TimeUnit.MILLISECONDS) + " ms");
    System.out.println("Scan metrics:\n" + metrics.getMetricsMap());
    System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")");
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughput) + "B/s");
    System.out.println("total rows  : " + numRows);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputRows) + " rows/s");
    System.out.println("total cells : " + numCells);
    System.out.println("throughput  : " + StringUtils.humanReadableInt((long) throughputCells) + " cells/s");
}

Example 5 with ScanMetrics

use of org.apache.hadoop.hbase.client.metrics.ScanMetrics in project hbase by apache.

the class TableRecordReaderImpl method updateCounters.

/**
 * If hbase runs on new version of mapreduce, RecordReader has access to
 * counters thus can update counters based on scanMetrics.
 * If hbase runs on old version of mapreduce, it won't be able to get
 * access to counters and TableRecorderReader can't update counter values.
 */
private void updateCounters() {
    ScanMetrics scanMetrics = scanner.getScanMetrics();
    if (scanMetrics == null) {
        return;
    }
    updateCounters(scanMetrics, numRestarts, context, numStale);
}