Examples with DFSInputStream org.apache.hadoop.hdfs.DFSInputStream used on opensource projects

Example 1 with DFSInputStream

use of org.apache.hadoop.hdfs.DFSInputStream in project hadoop by apache.

the class DFSClientCache method inputStreamLoader.

private CacheLoader<DFSInputStreamCaheKey, FSDataInputStream> inputStreamLoader() {
    return new CacheLoader<DFSInputStreamCaheKey, FSDataInputStream>() {

        @Override
        public FSDataInputStream load(DFSInputStreamCaheKey key) throws Exception {
            DFSClient client = getDfsClient(key.userId);
            DFSInputStream dis = client.open(key.inodePath);
            return client.createWrappedInputStream(dis);
        }
    };
}

Example 2 with DFSInputStream

use of org.apache.hadoop.hdfs.DFSInputStream in project SSM by Intel-bigdata.

the class ErasureCodingBase method convert.

protected void convert(HdfsFileStatus srcFileStatus) throws ActionException {
    DFSInputStream in = null;
    DFSOutputStream out = null;
    try {
        long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
        in = dfsClient.open(srcPath, bufferSize, true);
        short replication = (short) conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT);
        // use the same FsPermission as srcPath
        FsPermission permission = srcFileStatus.getPermission();
        out = dfsClient.create(ecTmpPath, permission, EnumSet.of(CreateFlag.CREATE), true, replication, blockSize, null, bufferSize, null, null, ecPolicyName);
        // Keep storage policy according with original file except UNDEF storage policy
        String storagePolicyName = dfsClient.getStoragePolicy(srcPath).getName();
        if (!storagePolicyName.equals("UNDEF")) {
            dfsClient.setStoragePolicy(ecTmpPath, storagePolicyName);
        }
        long bytesRemaining = srcFileStatus.getLen();
        byte[] buf = new byte[bufferSize];
        while (bytesRemaining > 0L) {
            int bytesToRead = (int) (bytesRemaining < (long) buf.length ? bytesRemaining : (long) buf.length);
            int bytesRead = in.read(buf, 0, bytesToRead);
            if (bytesRead == -1) {
                break;
            }
            out.write(buf, 0, bytesRead);
            bytesRemaining -= (long) bytesRead;
            this.progress = (float) (srcFileStatus.getLen() - bytesRemaining) / srcFileStatus.getLen();
        }
    } catch (Exception ex) {
        throw new ActionException(ex);
    } finally {
        try {
            if (in != null) {
                in.close();
            }
            if (out != null) {
                out.close();
            }
        } catch (IOException ex) {
            LOG.error("IOException occurred when closing DFSInputStream or DFSOutputStream!");
        }
    }
}

Example 3 with DFSInputStream

use of org.apache.hadoop.hdfs.DFSInputStream in project SSM by Intel-bigdata.

the class TestCompressDecompress method testCompressedFileRandomRead.

// @Test(timeout = 90000)
// public void testCompressEmptyFile() throws Exception {
// waitTillSSMExitSafeMode();
// 
// // initDB();
// String fileName = "/ssm/compression/file2";
// prepareFile(fileName, 0);
// MetaStore metaStore = ssm.getMetaStore();
// 
// int bufSize = 1024 * 1024;
// CmdletManager cmdletManager = ssm.getCmdletManager();
// long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName
// + " -bufSize " + bufSize + " -compressImpl " + compressionImpl);
// 
// waitTillActionDone(cmdId);
// FileState fileState = metaStore.getFileState(fileName);
// while (!fileState.getFileType().equals(FileState.FileType.COMPRESSION)) {
// Thread.sleep(200);
// fileState = metaStore.getFileState(fileName);
// }
// 
// // metastore  test
// //    Assert.assertEquals(FileState.FileType.COMPRESSION, fileState.getFileType());
// Assert.assertEquals(FileState.FileStage.DONE, fileState.getFileStage());
// Assert.assertTrue(fileState instanceof CompressionFileState);
// CompressionFileState compressionFileState = (CompressionFileState) fileState;
// Assert.assertEquals(fileName, compressionFileState.getPath());
// Assert.assertEquals(bufSize, compressionFileState.getBufferSize());
// Assert.assertEquals(compressionImpl, compressionFileState.getCompressionImpl());
// Assert.assertEquals(0, compressionFileState.getOriginalLength());
// Assert.assertEquals(0, compressionFileState.getCompressedLength());
// 
// // File length test
// Assert.assertEquals(0, dfsClient.getFileInfo(fileName).getLen());
// }
@Test
public void testCompressedFileRandomRead() throws Exception {
    // if (!loadedNative()) {
    // return;
    // }
    waitTillSSMExitSafeMode();
    // initDB();
    int arraySize = 1024 * 1024 * 8;
    String fileName = "/ssm/compression/file3";
    byte[] bytes = prepareFile(fileName, arraySize);
    int bufSize = 1024 * 1024;
    CmdletManager cmdletManager = ssm.getCmdletManager();
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    // Test random read
    Random rnd = new Random(System.currentTimeMillis());
    DFSInputStream dfsInputStream = smartDFSClient.open(fileName);
    int randomReadSize = 500;
    byte[] randomReadBuffer = new byte[randomReadSize];
    for (int i = 0; i < 5; i++) {
        int pos = rnd.nextInt(arraySize - 500);
        byte[] subBytes = Arrays.copyOfRange(bytes, pos, pos + 500);
        dfsInputStream.seek(pos);
        Assert.assertEquals(pos, dfsInputStream.getPos());
        int off = 0;
        while (off < randomReadSize) {
            int len = dfsInputStream.read(randomReadBuffer, off, randomReadSize - off);
            off += len;
        }
        Assert.assertArrayEquals(subBytes, randomReadBuffer);
        Assert.assertEquals(pos + 500, dfsInputStream.getPos());
    }
}

Example 4 with DFSInputStream

use of org.apache.hadoop.hdfs.DFSInputStream in project SSM by Intel-bigdata.

the class SmartDFSClient method open.

@Override
public DFSInputStream open(String src, int buffersize, boolean verifyChecksum) throws IOException {
    DFSInputStream is = super.open(src, buffersize, verifyChecksum);
    if (is.getFileLength() == 0) {
        is.close();
        FileState fileState = getFileState(src);
        if (fileState.getFileStage().equals(FileState.FileStage.PROCESSING)) {
            throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + fileState.getFileType());
        }
        is = SmartInputStreamFactory.create(this, src, verifyChecksum, fileState);
    } else {
        is.close();
        FileState fileState = getFileState(src);
        if (fileState.getFileStage().equals(FileState.FileStage.PROCESSING)) {
            throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + fileState.getFileType());
        }
        is = SmartInputStreamFactory.create(this, src, verifyChecksum, fileState);
    }
    reportFileAccessEvent(src);
    return is;
}

Example 5 with DFSInputStream

use of org.apache.hadoop.hdfs.DFSInputStream in project SSM by Intel-bigdata.

the class SmallFileUncompactAction method execute.

@Override
protected void execute() throws Exception {
    // Set hdfs client by DFSClient rather than SmartDFSClient
    this.setDfsClient(HadoopUtil.getDFSClient(HadoopUtil.getNameNodeUri(conf), conf));
    // Get small file list
    if (smallFiles == null || smallFiles.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid small files: %s.", smallFiles));
    }
    ArrayList<String> smallFileList = new Gson().fromJson(smallFiles, new TypeToken<ArrayList<String>>() {
    }.getType());
    if (smallFileList == null || smallFileList.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid small files: %s.", smallFiles));
    }
    // Get container file path
    if (containerFile == null || containerFile.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid container file: %s.", containerFile));
    }
    appendLog(String.format("Action starts at %s : uncompact small files.", Utils.getFormatedCurrentTime()));
    for (String smallFile : smallFileList) {
        if ((smallFile != null) && !smallFile.isEmpty() && dfsClient.exists(smallFile)) {
            DFSInputStream in = null;
            OutputStream out = null;
            try {
                // Get compact input stream
                in = smartDFSClient.open(smallFile);
                // Save original metadata of small file and delete original small file
                HdfsFileStatus fileStatus = dfsClient.getFileInfo(smallFile);
                Map<String, byte[]> xAttr = dfsClient.getXAttrs(smallFile);
                dfsClient.delete(smallFile, false);
                // Create new small file
                out = dfsClient.create(smallFile, true);
                // Copy contents to original small file
                IOUtils.copyBytes(in, out, 4096);
                // Reset file meta data
                resetFileMeta(smallFile, fileStatus, xAttr);
                // Set status and update log
                this.status = (smallFileList.indexOf(smallFile) + 1.0f) / smallFileList.size();
                appendLog(String.format("Uncompact %s successfully.", smallFile));
            } finally {
                if (in != null) {
                    in.close();
                }
                if (out != null) {
                    out.close();
                }
            }
        }
    }
    dfsClient.delete(containerFile, false);
    appendLog(String.format("Uncompact all the small files of %s successfully.", containerFile));
}