Examples with SmartFilePermission org.smartdata.SmartFilePermission used on opensource projects

Example 1 with SmartFilePermission

use of org.smartdata.SmartFilePermission in project SSM by Intel-bigdata.

the class SmallFileScheduler method getContainerFilePermission.

/**
 * Get container file info according to action arguments and meta store.
 */
private SmartFilePermission getContainerFilePermission(ActionInfo actionInfo, String containerFilePath) throws MetaStoreException, IllegalArgumentException {
    // Get container file permission from the argument of this action
    String containerFilePermissionArg = actionInfo.getArgs().get(SmallFileCompactAction.CONTAINER_FILE_PERMISSION);
    SmartFilePermission containerFilePermissionFromArg = null;
    if (containerFilePermissionArg != null && !containerFilePermissionArg.isEmpty()) {
        containerFilePermissionFromArg = new Gson().fromJson(containerFilePermissionArg, new TypeToken<SmartFilePermission>() {
        }.getType());
    }
    // Get container file permission from meta store
    SmartFilePermission containerFilePermissionFromMeta = null;
    FileInfo containerFileInfo = metaStore.getFile(containerFilePath);
    if (containerFileInfo != null) {
        containerFilePermissionFromMeta = new SmartFilePermission(containerFileInfo);
    }
    // Get container file permission
    SmartFilePermission containerFilePermission;
    if (containerFilePermissionFromArg == null || containerFilePermissionFromMeta == null) {
        containerFilePermission = (containerFilePermissionFromArg == null) ? containerFilePermissionFromMeta : containerFilePermissionFromArg;
    } else {
        if (containerFilePermissionFromArg.equals(containerFilePermissionFromMeta)) {
            containerFilePermission = containerFilePermissionFromArg;
        } else {
            throw new IllegalArgumentException("Illegal container file permission argument.");
        }
    }
    return containerFilePermission;
}

Example 2 with SmartFilePermission

use of org.smartdata.SmartFilePermission in project SSM by Intel-bigdata.

the class SmallFileScheduler method getCompactScheduleResult.

/**
 * Get compact action schedule result according to action info.
 */
private ScheduleResult getCompactScheduleResult(ActionInfo actionInfo) {
    // Get container file and small file list of this action
    String containerFilePath = getContainerFile(actionInfo);
    ArrayList<String> smallFileList = new Gson().fromJson(actionInfo.getArgs().get(HdfsAction.FILE_PATH), new TypeToken<ArrayList<String>>() {
    }.getType());
    // Check if container file is locked and retry
    if (containerFileLock.contains(containerFilePath)) {
        return ScheduleResult.RETRY;
    } else {
        // Check if the small file list is valid
        if (!checkIfValidSmallFiles(smallFileList)) {
            actionInfo.setResult("Small file list is invalid.");
            return ScheduleResult.FAIL;
        }
        // Get container file permission
        SmartFilePermission containerFilePermission;
        try {
            containerFilePermission = getContainerFilePermission(actionInfo, containerFilePath);
        } catch (MetaStoreException e1) {
            actionInfo.setResult(String.format("Failed to get file info of the container file %s for %s.", containerFilePath, e1.toString()));
            return ScheduleResult.FAIL;
        } catch (IllegalArgumentException e2) {
            actionInfo.setResult(e2.getMessage());
            return ScheduleResult.FAIL;
        }
        // Get first small file info
        FileInfo firstFileInfo;
        SmartFilePermission firstFilePermission;
        try {
            firstFileInfo = metaStore.getFile(smallFileList.get(0));
            firstFilePermission = new SmartFilePermission(firstFileInfo);
        } catch (MetaStoreException e) {
            actionInfo.setResult(String.format("Failed to get first file info: %s.", containerFilePath));
            return ScheduleResult.FAIL;
        }
        // and its permission is null
        if (containerFilePermission == null) {
            Map<String, String> args = new HashMap<>(3);
            args.put(SmallFileCompactAction.CONTAINER_FILE, getContainerFile(actionInfo));
            args.put(SmallFileCompactAction.FILE_PATH, new Gson().toJson(smallFileList));
            args.put(SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(firstFilePermission));
            actionInfo.setArgs(args);
        } else {
            if (!containerFilePermission.equals(firstFilePermission)) {
                actionInfo.setResult(String.format("Container file %s has different permission with %s.", containerFilePath, firstFileInfo.getPath()));
                return ScheduleResult.FAIL;
            }
        }
        // Lock container file and small files
        containerFileLock.add(containerFilePath);
        compactSmallFileLock.addAll(smallFileList);
        afterSchedule(actionInfo);
        return ScheduleResult.SUCCESS;
    }
}

Example 3 with SmartFilePermission

use of org.smartdata.SmartFilePermission in project SSM by Intel-bigdata.

the class SmallFileScheduler method checkIfValidSmallFiles.

/**
 * Check if the small file list is valid.
 */
private boolean checkIfValidSmallFiles(List<String> smallFileList) {
    for (String smallFile : smallFileList) {
        if (smallFile == null || smallFile.isEmpty()) {
            LOG.error("Illegal small file path: {}", smallFile);
            return false;
        } else if (compactSmallFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is locked.", smallFile));
            return false;
        } else if (handlingSmallFileCache.contains(smallFile)) {
            LOG.error(String.format("%s is being handling.", smallFile));
            return false;
        } else if (containerFileCache.contains(smallFile) || containerFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is container file.", smallFile));
            return false;
        }
    }
    // Get small file info list and file state map from meta store.
    List<FileInfo> fileInfos;
    Map<String, FileState> fileStateMap;
    try {
        fileInfos = metaStore.getFilesByPaths(smallFileList);
        fileStateMap = metaStore.getFileStates(smallFileList);
    } catch (MetaStoreException e) {
        LOG.error("Failed to get file states of small files.", e);
        return false;
    }
    // Get small file info map
    Map<String, FileInfo> fileInfoMap = new HashMap<>();
    for (FileInfo fileInfo : fileInfos) {
        fileInfoMap.put(fileInfo.getPath(), fileInfo);
    }
    // Check if the permission of small file is same,
    // and all the small files exist
    FileInfo firstFileInfo = null;
    for (String smallFile : smallFileList) {
        FileInfo fileInfo = fileInfoMap.get(smallFile);
        if (fileInfo != null) {
            if (firstFileInfo == null) {
                firstFileInfo = fileInfo;
            } else {
                if (!(new SmartFilePermission(firstFileInfo)).equals(new SmartFilePermission(fileInfo))) {
                    LOG.debug(String.format("%s has different file permission with %s.", firstFileInfo.getPath(), fileInfo.getPath()));
                    return false;
                }
            }
        } else {
            LOG.debug("{} is not exist!!!", smallFile);
            return false;
        }
    }
    // Check if the state of small file is NORMAL
    for (Map.Entry<String, FileState> entry : fileStateMap.entrySet()) {
        String smallFile = entry.getKey();
        FileState.FileType smallFileType = entry.getValue().getFileType();
        if (smallFileType != FileState.FileType.NORMAL) {
            LOG.debug(String.format("%s has invalid file state %s for small file compact.", smallFile, smallFileType.toString()));
            return false;
        }
    }
    return true;
}

Example 4 with SmartFilePermission

use of org.smartdata.SmartFilePermission in project SSM by Intel-bigdata.

the class SmallFilePlugin method preSubmitCmdletDescriptor.

@Override
public CmdletDescriptor preSubmitCmdletDescriptor(final RuleInfo ruleInfo, TranslateResult tResult, CmdletDescriptor descriptor) {
    for (int i = 0; i < descriptor.getActionSize(); i++) {
        if (COMPACT_ACTION_NAME.equals(descriptor.getActionName(i))) {
            String smallFiles = descriptor.getActionArgs(i).get(HdfsAction.FILE_PATH);
            if (smallFiles != null && !smallFiles.isEmpty()) {
                // Check if small file list is empty
                ArrayList<String> smallFileList = new Gson().fromJson(smallFiles, new TypeToken<ArrayList<String>>() {
                }.getType());
                if (smallFileList == null || smallFileList.isEmpty()) {
                    continue;
                }
                // Get the first small file info
                String firstFile = smallFileList.get(0);
                FileInfo firstFileInfo;
                if (firstFileInfoCache.containsKey(firstFile)) {
                    firstFileInfo = firstFileInfoCache.get(firstFile);
                } else {
                    try {
                        firstFileInfo = metaStore.getFile(firstFile);
                        if (firstFileInfo == null) {
                            LOG.debug("{} is not exist!!!", firstFile);
                            continue;
                        }
                    } catch (MetaStoreException e) {
                        LOG.error(String.format("Failed to get file info of: %s.", firstFile), e);
                        continue;
                    }
                }
                // Get valid compact action arguments
                SmartFilePermission firstFilePermission = new SmartFilePermission(firstFileInfo);
                String firstFileDir = firstFile.substring(0, firstFile.lastIndexOf("/") + 1);
                CompactActionArgs args = getCompactActionArgs(ruleInfo, firstFileDir, firstFilePermission, smallFileList);
                // Set container file path and its permission, file path of this action
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE, args.containerFile);
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(args.containerFilePermission));
                descriptor.addActionArg(i, HdfsAction.FILE_PATH, new Gson().toJson(args.smartFiles));
            }
        }
    }
    return descriptor;
}

Example 5 with SmartFilePermission

use of org.smartdata.SmartFilePermission in project SSM by Intel-bigdata.

the class SmallFilePlugin method getCompactActionArgs.

/**
 * Get valid compact action arguments.
 */
private CompactActionArgs getCompactActionArgs(RuleInfo ruleInfo, String firstFileDir, SmartFilePermission firstFilePermission, List<String> smallFileList) {
    Map<String, FileInfo> containerFileMap = containerFileInfoCache.get(ruleInfo);
    for (Iterator<Map.Entry<String, FileInfo>> iter = containerFileMap.entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<String, FileInfo> entry = iter.next();
        String containerFilePath = entry.getKey();
        FileInfo containerFileInfo = entry.getValue();
        iter.remove();
        // Get compact action arguments
        String containerFileDir = containerFilePath.substring(0, containerFilePath.lastIndexOf("/") + 1);
        if (firstFileDir.equals(containerFileDir) && firstFilePermission.equals(new SmartFilePermission(containerFileInfo))) {
            List<String> validSmallFiles;
            try {
                validSmallFiles = getValidSmallFiles(containerFileInfo, smallFileList);
            } catch (MetaStoreException e) {
                LOG.error("Failed to get file info of small files.", e);
                continue;
            }
            if (validSmallFiles != null) {
                return new CompactActionArgs(containerFilePath, null, validSmallFiles);
            }
        }
    }
    return genCompactActionArgs(firstFileDir, firstFilePermission, smallFileList);
}