Examples with MetricsRecordBuilder org.apache.hadoop.metrics2.MetricsRecordBuilder used on opensource projects

Example 61 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestNameNodeMetrics method testExcessBlocks.

/** Create excess blocks by reducing the replication factor for
   * for a file and ensure metrics reflects it
   */
@Test
public void testExcessBlocks() throws Exception {
    Path file = getTestPath("testExcessBlocks");
    createFile(file, 100, (short) 2);
    NameNodeAdapter.setReplication(namesystem, file.toString(), (short) 1);
    MetricsRecordBuilder rb = getMetrics(NS_METRICS);
    assertGauge("ExcessBlocks", 1L, rb);
    // verify ExcessBlocks metric is decremented and
    // excessReplicateMap is cleared after deleting a file
    fs.delete(file, true);
    rb = getMetrics(NS_METRICS);
    assertGauge("ExcessBlocks", 0L, rb);
    assertEquals(0L, bm.getExcessBlocksCount());
}

Example 62 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestNameNodeMetrics method testCorruptBlock.

/** Corrupt a block and ensure metrics reflects it */
@Test
public void testCorruptBlock() throws Exception {
    // Create a file with single block with two replicas
    final Path file = getTestPath("testCorruptBlock");
    createFile(file, 100, (short) 2);
    // can be fixed
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    // Corrupt first replica of the block
    LocatedBlock block = NameNodeAdapter.getBlockLocations(cluster.getNameNode(), file.toString(), 0, 1).get(0);
    cluster.getNamesystem().writeLock();
    try {
        bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], "STORAGE_ID", "TEST");
    } finally {
        cluster.getNamesystem().writeUnlock();
    }
    BlockManagerTestUtil.getComputedDatanodeWork(bm);
    MetricsRecordBuilder rb = getMetrics(NS_METRICS);
    assertGauge("CorruptBlocks", 1L, rb);
    assertGauge("PendingReplicationBlocks", 1L, rb);
    fs.delete(file, true);
    // During the file deletion, both BlockManager#corruptReplicas and
    // BlockManager#pendingReplications will be updated, i.e., the records
    // for the blocks of the deleted file will be removed from both
    // corruptReplicas and pendingReplications. The corresponding
    // metrics (CorruptBlocks and PendingReplicationBlocks) will only be updated
    // when BlockManager#computeDatanodeWork is run where the
    // BlockManager#updateState is called. And in
    // BlockManager#computeDatanodeWork the metric ScheduledReplicationBlocks
    // will also be updated.
    rb = waitForDnMetricValue(NS_METRICS, "CorruptBlocks", 0L);
    assertGauge("PendingReplicationBlocks", 0L, rb);
    assertGauge("ScheduledReplicationBlocks", 0L, rb);
}

Example 63 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestNameNodeMetrics method testSyncAndBlockReportMetric.

/**
   * Tests that the sync and block report metrics get updated on cluster
   * startup.
   */
@Test
public void testSyncAndBlockReportMetric() throws Exception {
    MetricsRecordBuilder rb = getMetrics(NN_METRICS);
    // We have one sync when the cluster starts up, just opening the journal
    assertCounter("SyncsNumOps", 1L, rb);
    // Each datanode reports in when the cluster comes up
    assertCounter("BlockReportNumOps", (long) DATANODE_COUNT * cluster.getStoragesPerDatanode(), rb);
    // Sleep for an interval+slop to let the percentiles rollover
    Thread.sleep((PERCENTILES_INTERVAL + 1) * 1000);
    // Check that the percentiles were updated
    assertQuantileGauges("Syncs1s", rb);
    assertQuantileGauges("BlockReport1s", rb);
}

Example 64 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestNameNodeMetrics method testReadWriteOps.

/**
   * Test NN ReadOps Count and WriteOps Count
   */
@Test
public void testReadWriteOps() throws Exception {
    MetricsRecordBuilder rb = getMetrics(NN_METRICS);
    long startWriteCounter = MetricsAsserts.getLongCounter("TransactionsNumOps", rb);
    Path file1_Path = new Path(TEST_ROOT_DIR_PATH, "ReadData.dat");
    //Perform create file operation
    createFile(file1_Path, 1024, (short) 2);
    // Perform read file operation on earlier created file
    readFile(fs, file1_Path);
    MetricsRecordBuilder rbNew = getMetrics(NN_METRICS);
    assertTrue(MetricsAsserts.getLongCounter("TransactionsNumOps", rbNew) > startWriteCounter);
}

Example 65 with MetricsRecordBuilder

use of org.apache.hadoop.metrics2.MetricsRecordBuilder in project hadoop by apache.

the class TestNameNodeMetrics method testMissingBlock.

/** Test to ensure metrics reflects missing blocks */
@Test
public void testMissingBlock() throws Exception {
    // Create a file with single block with two replicas
    Path file = getTestPath("testMissingBlocks");
    createFile(file, 100, (short) 1);
    // Corrupt the only replica of the block to result in a missing block
    LocatedBlock block = NameNodeAdapter.getBlockLocations(cluster.getNameNode(), file.toString(), 0, 1).get(0);
    cluster.getNamesystem().writeLock();
    try {
        bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], "STORAGE_ID", "TEST");
    } finally {
        cluster.getNamesystem().writeUnlock();
    }
    // Wait for block to be marked corrupt
    Thread.sleep(1000);
    MetricsRecordBuilder rb = getMetrics(NS_METRICS);
    assertGauge("UnderReplicatedBlocks", 1L, rb);
    assertGauge("MissingBlocks", 1L, rb);
    assertGauge("MissingReplOneBlocks", 1L, rb);
    fs.delete(file, true);
    waitForDnMetricValue(NS_METRICS, "UnderReplicatedBlocks", 0L);
}