Examples with StreamSlowMonitor org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor used on opensource projects

Example 1 with StreamSlowMonitor

use of org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor in project hbase by apache.

the class TestExcludeDatanodeManager method testExcludeSlowDNByProcessTime.

@Test
public void testExcludeSlowDNByProcessTime() {
    Configuration conf = HBaseConfiguration.create();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(conf);
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    DatanodeInfo datanodeInfo = new DatanodeInfo.DatanodeInfoBuilder().setIpAddr("0.0.0.0").setHostName("hostname1").setDatanodeUuid("uuid1").setXferPort(111).setInfoPort(222).setInfoSecurePort(333).setIpcPort(444).setNetworkLocation("location1").build();
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    assertTrue(excludeDatanodeManager.getExcludeDNs().containsKey(datanodeInfo));
}

Example 2 with StreamSlowMonitor

use of org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor in project hbase by apache.

the class TestExcludeDatanodeManager method testExcludeSlowDNBySpeed.

@Test
public void testExcludeSlowDNBySpeed() {
    Configuration conf = HBaseConfiguration.create();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(conf);
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    DatanodeInfo datanodeInfo = new DatanodeInfo.DatanodeInfoBuilder().setIpAddr("0.0.0.0").setHostName("hostname1").setDatanodeUuid("uuid1").setXferPort(111).setInfoPort(222).setInfoSecurePort(333).setIpcPort(444).setNetworkLocation("location1").build();
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    assertTrue(excludeDatanodeManager.getExcludeDNs().containsKey(datanodeInfo));
}

Example 3 with StreamSlowMonitor

use of org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor in project hbase by apache.

the class TestFanOutOneBlockAsyncDFSOutput method testExcludeFailedConnectToDatanode.

@Test
public void testExcludeFailedConnectToDatanode() throws IOException, ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException, NoSuchFieldException {
    Field xceiverServerDaemonField = DataNode.class.getDeclaredField("dataXceiverServer");
    xceiverServerDaemonField.setAccessible(true);
    Class<?> xceiverServerClass = Class.forName("org.apache.hadoop.hdfs.server.datanode.DataXceiverServer");
    Method numPeersMethod = xceiverServerClass.getDeclaredMethod("getNumPeers");
    numPeersMethod.setAccessible(true);
    // make one datanode broken
    DataNodeProperties dnProp = CLUSTER.stopDataNode(0);
    Path f = new Path("/test");
    EventLoop eventLoop = EVENT_LOOP_GROUP.next();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(HBaseConfiguration.create());
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    try (FanOutOneBlockAsyncDFSOutput output = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, streamSlowDNsMonitor)) {
        // should exclude the dead dn when retry so here we only have 2 DNs in pipeline
        assertEquals(2, output.getPipeline().length);
        assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    } finally {
        CLUSTER.restartDataNode(dnProp);
    }
}