Examples with MD5MD5CRC32FileChecksum org.apache.hadoop.fs.MD5MD5CRC32FileChecksum used on opensource projects

Example 11 with MD5MD5CRC32FileChecksum

use of org.apache.hadoop.fs.MD5MD5CRC32FileChecksum in project SSM by Intel-bigdata.

the class CheckSumAction method execute.

@Override
protected void execute() throws Exception {
    // HadoopUtil.getNameNodeUri(conf), conf));
    if (fileName == null) {
        throw new IllegalArgumentException("Please specify file path!");
    }
    if (fileName.charAt(fileName.length() - 1) == '*') {
        DirectoryListing listing = dfsClient.listPaths(fileName.substring(0, fileName.length() - 1), HdfsFileStatus.EMPTY_NAME);
        HdfsFileStatus[] fileList = listing.getPartialListing();
        for (HdfsFileStatus fileStatus : fileList) {
            String file1 = fileStatus.getFullPath(new Path(fileName.substring(0, fileName.length() - 1))).toString();
            HdfsFileStatus fileStatus1 = dfsClient.getFileInfo(file1);
            long length = fileStatus1.getLen();
            MD5MD5CRC32FileChecksum md5 = dfsClient.getFileChecksum(file1, length);
            ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
            DataOutputStream dataStream = new DataOutputStream(byteStream);
            md5.write(dataStream);
            byte[] bytes = byteStream.toByteArray();
            appendLog(String.format("%s\t%s\t%s", file1, md5.getAlgorithmName(), byteArray2HexString(bytes)));
        }
        return;
    }
    HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName);
    if (fileStatus != null) {
        if (fileStatus.isDir()) {
            appendResult("This is a directory which has no checksum result!");
            appendLog("This is a directory which has no checksum result!");
            return;
        }
    }
    long length = fileStatus.getLen();
    MD5MD5CRC32FileChecksum md5 = dfsClient.getFileChecksum(fileName, length);
    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
    DataOutputStream dataStream = new DataOutputStream(byteStream);
    md5.write(dataStream);
    byte[] bytes = byteStream.toByteArray();
    appendLog(String.format("%s\t%s\t%s", fileName, md5.getAlgorithmName(), byteArray2HexString(bytes)));
}