Examples with ExcludeDatanodeManager org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager used on opensource projects

Example 1 with ExcludeDatanodeManager

use of org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputHelper method createOutput.

private static FanOutOneBlockAsyncDFSOutput createOutput(DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, EventLoopGroup eventLoopGroup, Class<? extends Channel> channelClass, StreamSlowMonitor monitor) throws IOException {
    Configuration conf = dfs.getConf();
    DFSClient client = dfs.getClient();
    String clientName = client.getClientName();
    ClientProtocol namenode = client.getNamenode();
    int createMaxRetries = conf.getInt(ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES, DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES);
    ExcludeDatanodeManager excludeDatanodeManager = monitor.getExcludeDatanodeManager();
    Set<DatanodeInfo> toExcludeNodes = new HashSet<>(excludeDatanodeManager.getExcludeDNs().keySet());
    for (int retry = 0; ; retry++) {
        LOG.debug("When create output stream for {}, exclude list is {}, retry={}", src, toExcludeNodes, retry);
        HdfsFileStatus stat;
        try {
            stat = FILE_CREATOR.create(namenode, src, FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)), clientName, getCreateFlags(overwrite), createParent, replication, blockSize, CryptoProtocolVersion.supported());
        } catch (Exception e) {
            if (e instanceof RemoteException) {
                throw (RemoteException) e;
            } else {
                throw new NameNodeException(e);
            }
        }
        beginFileLease(client, stat.getFileId());
        boolean succ = false;
        LocatedBlock locatedBlock = null;
        List<Future<Channel>> futureList = null;
        try {
            DataChecksum summer = createChecksum(client);
            locatedBlock = namenode.addBlock(src, client.getClientName(), null, toExcludeNodes.toArray(new DatanodeInfo[0]), stat.getFileId(), null, null);
            Map<Channel, DatanodeInfo> datanodes = new IdentityHashMap<>();
            futureList = connectToDataNodes(conf, client, clientName, locatedBlock, 0L, 0L, PIPELINE_SETUP_CREATE, summer, eventLoopGroup, channelClass);
            for (int i = 0, n = futureList.size(); i < n; i++) {
                DatanodeInfo datanodeInfo = locatedBlock.getLocations()[i];
                try {
                    datanodes.put(futureList.get(i).syncUninterruptibly().getNow(), datanodeInfo);
                } catch (Exception e) {
                    // exclude the broken DN next time
                    toExcludeNodes.add(datanodeInfo);
                    excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "connect error");
                    throw e;
                }
            }
            Encryptor encryptor = createEncryptor(conf, stat, client);
            FanOutOneBlockAsyncDFSOutput output = new FanOutOneBlockAsyncDFSOutput(conf, dfs, client, namenode, clientName, src, stat.getFileId(), locatedBlock, encryptor, datanodes, summer, ALLOC, monitor);
            succ = true;
            return output;
        } catch (RemoteException e) {
            LOG.warn("create fan-out dfs output {} failed, retry = {}", src, retry, e);
            if (shouldRetryCreate(e)) {
                if (retry >= createMaxRetries) {
                    throw e.unwrapRemoteException();
                }
            } else {
                throw e.unwrapRemoteException();
            }
        } catch (IOException e) {
            LOG.warn("create fan-out dfs output {} failed, retry = {}", src, retry, e);
            if (retry >= createMaxRetries) {
                throw e;
            }
            // overwrite the old broken file.
            overwrite = true;
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(100, retry));
            } catch (InterruptedException ie) {
                throw new InterruptedIOException();
            }
        } finally {
            if (!succ) {
                if (futureList != null) {
                    for (Future<Channel> f : futureList) {
                        f.addListener(new FutureListener<Channel>() {

                            @Override
                            public void operationComplete(Future<Channel> future) throws Exception {
                                if (future.isSuccess()) {
                                    future.getNow().close();
                                }
                            }
                        });
                    }
                }
                endFileLease(client, stat.getFileId());
            }
        }
    }
}

Example 2 with ExcludeDatanodeManager

use of org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager in project hbase by apache.

the class TestExcludeDatanodeManager method testExcludeSlowDNByProcessTime.

@Test
public void testExcludeSlowDNByProcessTime() {
    Configuration conf = HBaseConfiguration.create();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(conf);
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    DatanodeInfo datanodeInfo = new DatanodeInfo.DatanodeInfoBuilder().setIpAddr("0.0.0.0").setHostName("hostname1").setDatanodeUuid("uuid1").setXferPort(111).setInfoPort(222).setInfoSecurePort(333).setIpcPort(444).setNetworkLocation("location1").build();
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 5000, 7000, System.currentTimeMillis() - 7000, 0);
    assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    assertTrue(excludeDatanodeManager.getExcludeDNs().containsKey(datanodeInfo));
}

Example 3 with ExcludeDatanodeManager

use of org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager in project hbase by apache.

the class TestExcludeDatanodeManager method testExcludeSlowDNBySpeed.

@Test
public void testExcludeSlowDNBySpeed() {
    Configuration conf = HBaseConfiguration.create();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(conf);
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    DatanodeInfo datanodeInfo = new DatanodeInfo.DatanodeInfoBuilder().setIpAddr("0.0.0.0").setHostName("hostname1").setDatanodeUuid("uuid1").setXferPort(111).setInfoPort(222).setInfoSecurePort(333).setIpcPort(444).setNetworkLocation("location1").build();
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    streamSlowDNsMonitor.checkProcessTimeAndSpeed(datanodeInfo, 100000, 5100, System.currentTimeMillis() - 5100, 0);
    assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    assertTrue(excludeDatanodeManager.getExcludeDNs().containsKey(datanodeInfo));
}

Example 4 with ExcludeDatanodeManager

use of org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager in project hbase by apache.

the class TestFanOutOneBlockAsyncDFSOutput method testExcludeFailedConnectToDatanode.

@Test
public void testExcludeFailedConnectToDatanode() throws IOException, ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException, NoSuchFieldException {
    Field xceiverServerDaemonField = DataNode.class.getDeclaredField("dataXceiverServer");
    xceiverServerDaemonField.setAccessible(true);
    Class<?> xceiverServerClass = Class.forName("org.apache.hadoop.hdfs.server.datanode.DataXceiverServer");
    Method numPeersMethod = xceiverServerClass.getDeclaredMethod("getNumPeers");
    numPeersMethod.setAccessible(true);
    // make one datanode broken
    DataNodeProperties dnProp = CLUSTER.stopDataNode(0);
    Path f = new Path("/test");
    EventLoop eventLoop = EVENT_LOOP_GROUP.next();
    ExcludeDatanodeManager excludeDatanodeManager = new ExcludeDatanodeManager(HBaseConfiguration.create());
    StreamSlowMonitor streamSlowDNsMonitor = excludeDatanodeManager.getStreamSlowMonitor("testMonitor");
    assertEquals(0, excludeDatanodeManager.getExcludeDNs().size());
    try (FanOutOneBlockAsyncDFSOutput output = FanOutOneBlockAsyncDFSOutputHelper.createOutput(FS, f, true, false, (short) 3, FS.getDefaultBlockSize(), eventLoop, CHANNEL_CLASS, streamSlowDNsMonitor)) {
        // should exclude the dead dn when retry so here we only have 2 DNs in pipeline
        assertEquals(2, output.getPipeline().length);
        assertEquals(1, excludeDatanodeManager.getExcludeDNs().size());
    } finally {
        CLUSTER.restartDataNode(dnProp);
    }
}