Examples with CompactFileState org.smartdata.model.CompactFileState used on opensource projects

Example 11 with CompactFileState

use of org.smartdata.model.CompactFileState in project SSM by Intel-bigdata.

the class SmartDFSClient method getLocatedBlocks.

@Override
public LocatedBlocks getLocatedBlocks(String src, long start) throws IOException {
    LocatedBlocks locatedBlocks = super.getLocatedBlocks(src, start);
    if (!CALLER_CLASS.equals(Thread.currentThread().getStackTrace()[2].getClassName()) && locatedBlocks.getFileLength() == 0) {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompactFileState) {
            String containerFile = ((CompactFileState) fileState).getFileContainerInfo().getContainerFilePath();
            long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset();
            return super.getLocatedBlocks(containerFile, offset + start);
        }
    }
    return locatedBlocks;
}

Example 12 with CompactFileState

use of org.smartdata.model.CompactFileState in project SSM by Intel-bigdata.

the class SmallFileCompactAction method execute.

@Override
protected void execute() throws Exception {
    // Set hdfs client by DFSClient rather than SmartDFSClient
    this.setDfsClient(HadoopUtil.getDFSClient(HadoopUtil.getNameNodeUri(conf), conf));
    // Get small file list
    if (smallFiles == null || smallFiles.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid small files: %s.", smallFiles));
    }
    ArrayList<String> smallFileList = new Gson().fromJson(smallFiles, new TypeToken<ArrayList<String>>() {
    }.getType());
    if (smallFileList == null || smallFileList.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid small files: %s.", smallFiles));
    }
    // Get container file path
    if (containerFile == null || containerFile.isEmpty()) {
        throw new IllegalArgumentException(String.format("Invalid container file: %s.", containerFile));
    }
    // Get container file permission
    SmartFilePermission filePermission = null;
    if (containerFilePermission != null && !containerFilePermission.isEmpty()) {
        filePermission = new Gson().fromJson(containerFilePermission, new TypeToken<SmartFilePermission>() {
        }.getType());
    }
    appendLog(String.format("Action starts at %s : compact small files to %s.", Utils.getFormatedCurrentTime(), containerFile));
    // Get initial offset and output stream
    // Create container file and set permission if not exists
    long offset;
    OutputStream out;
    boolean isContainerFileExist = dfsClient.exists(containerFile);
    if (isContainerFileExist) {
        offset = dfsClient.getFileInfo(containerFile).getLen();
        out = CompatibilityHelperLoader.getHelper().getDFSClientAppend(dfsClient, containerFile, 64 * 1024, offset);
    } else {
        out = dfsClient.create(containerFile, true);
        if (filePermission != null) {
            dfsClient.setOwner(containerFile, filePermission.getOwner(), filePermission.getGroup());
            dfsClient.setPermission(containerFile, new FsPermission(filePermission.getPermission()));
        }
        offset = 0L;
    }
    List<CompactFileState> compactFileStates = new ArrayList<>();
    for (String smallFile : smallFileList) {
        if ((smallFile != null) && !smallFile.isEmpty() && dfsClient.exists(smallFile)) {
            HdfsDataOutputStream append = (HdfsDataOutputStream) CompatibilityHelperLoader.getHelper().getDFSClientAppend(dfsClient, smallFile, 1024);
            long fileLen = dfsClient.getFileInfo(smallFile).getLen();
            if (fileLen > 0) {
                try (InputStream in = dfsClient.open(smallFile)) {
                    // Copy bytes of small file to container file
                    IOUtils.copyBytes(in, out, 4096);
                    // Truncate small file, add file container info to XAttr
                    CompactFileState compactFileState = new CompactFileState(smallFile, new FileContainerInfo(containerFile, offset, fileLen));
                    append.close();
                    truncateAndSetXAttr(smallFile, compactFileState);
                    // Update compact file state map, offset, status, and log
                    compactFileStates.add(compactFileState);
                    offset += fileLen;
                    this.status = (smallFileList.indexOf(smallFile) + 1.0f) / smallFileList.size();
                    appendLog(String.format("Compact %s to %s successfully.", smallFile, containerFile));
                } catch (IOException e) {
                    // Close append, output streams and put compact file state map into action result
                    if (append != null) {
                        append.close();
                    }
                    if (out != null) {
                        out.close();
                        appendResult(new Gson().toJson(compactFileStates));
                    }
                    if (!isContainerFileExist && compactFileStates.isEmpty()) {
                        dfsClient.delete(containerFile, false);
                    }
                    throw e;
                }
            }
        }
    }
    appendResult(new Gson().toJson(compactFileStates));
    if (out != null) {
        out.close();
    }
    if (!isContainerFileExist && compactFileStates.isEmpty()) {
        dfsClient.delete(containerFile, false);
    }
    appendLog(String.format("Compact all the small files to %s successfully.", containerFile));
}

Example 13 with CompactFileState

use of org.smartdata.model.CompactFileState in project SSM by Intel-bigdata.

the class SmallFileScheduler method handleCompactActionResult.

/**
 * Handle compact action result.
 */
private void handleCompactActionResult(ActionInfo actionInfo) {
    // Get container file path, small files, result of this action
    String containerFilePath = getContainerFile(actionInfo);
    List<String> smallFileList = new Gson().fromJson(actionInfo.getArgs().get(HdfsAction.FILE_PATH), new TypeToken<ArrayList<String>>() {
    }.getType());
    List<CompactFileState> compactFileStates = new Gson().fromJson(actionInfo.getResult(), new TypeToken<ArrayList<CompactFileState>>() {
    }.getType());
    // handling small file cache
    if (compactFileStates != null && !compactFileStates.isEmpty()) {
        LOG.debug(String.format("Add container file %s into cache.", containerFilePath));
        containerFileCache.add(containerFilePath);
        for (CompactFileState compactFileState : compactFileStates) {
            handlingSmallFileCache.add(compactFileState.getPath());
            compactFileStateQueue.offer(compactFileState);
        }
    }
    // Remove locks of container file and small files
    containerFileLock.remove(containerFilePath);
    compactSmallFileLock.removeAll(smallFileList);
}

Example 14 with CompactFileState

use of org.smartdata.model.CompactFileState in project SSM by Intel-bigdata.

the class SmartFileSystem method isFileClosed.

@Override
public boolean isFileClosed(final Path src) throws IOException {
    boolean isFileClosed = super.isFileClosed(src);
    if (!isFileClosed) {
        FileState fileState = smartDFSClient.getFileState(getPathName(src));
        if (fileState instanceof CompactFileState) {
            String containerFile = ((CompactFileState) fileState).getFileContainerInfo().getContainerFilePath();
            isFileClosed = smartDFSClient.isFileClosed(containerFile);
        }
    }
    return isFileClosed;
}

Example 15 with CompactFileState

use of org.smartdata.model.CompactFileState in project SSM by Intel-bigdata.

the class SmartDFSClient method getFileChecksum.

@Override
public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) throws IOException {
    MD5MD5CRC32FileChecksum ret = super.getFileChecksum(src, length);
    if (ret.getChecksumOpt().getBytesPerChecksum() == 0) {
        FileState fileState = getFileState(src);
        if (fileState instanceof CompactFileState) {
            try {
                // Get original checksum for small file.
                byte[] bytes = getXAttr(src, SmartConstants.SMART_FILE_CHECKSUM_XATTR_NAME);
                ret = new MD5MD5CRC32FileChecksum();
                ret.readFields(new DataInputStream(new ByteArrayInputStream(bytes)));
            } catch (IOException e) {
                throw new IOException("Failed to get checksum for SSM Small File: " + e.getMessage());
            }
        }
    }
    return ret;
}