Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 16 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class MetaStore method listFileActions.

public List<DetailedFileAction> listFileActions(long rid, long start, long offset) throws MetaStoreException {
    if (mapStoragePolicyIdName == null) {
        updateCache();
    }
    List<ActionInfo> actionInfos = getActions(rid, start, offset);
    List<DetailedFileAction> detailedFileActions = new ArrayList<>();
    for (ActionInfo actionInfo : actionInfos) {
        DetailedFileAction detailedFileAction = new DetailedFileAction(actionInfo);
        String filePath = actionInfo.getArgs().get("-file");
        FileInfo fileInfo = getFile(filePath);
        if (fileInfo == null) {
            // LOG.debug("Namespace is not sync! File {} not in file table!", filePath);
            // Add a mock fileInfo
            fileInfo = new FileInfo(filePath, 0L, 0L, false, (short) 0, 0L, 0L, 0L, (short) 0, "root", "root", (byte) 0, (byte) 0);
        }
        detailedFileAction.setFileLength(fileInfo.getLength());
        detailedFileAction.setFilePath(filePath);
        if (actionInfo.getActionName().contains("allssd") || actionInfo.getActionName().contains("onessd") || actionInfo.getActionName().contains("archive") || actionInfo.getActionName().contains("alldisk") || actionInfo.getActionName().contains("onedisk") || actionInfo.getActionName().contains("ramdisk")) {
            detailedFileAction.setTarget(actionInfo.getActionName());
            detailedFileAction.setSrc(mapStoragePolicyIdName.get((int) fileInfo.getStoragePolicy()));
        } else {
            detailedFileAction.setSrc(actionInfo.getArgs().get("-src"));
            detailedFileAction.setTarget(actionInfo.getArgs().get("-dest"));
        }
        detailedFileActions.add(detailedFileAction);
    }
    return detailedFileActions;
}

Example 17 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestTableAggregator method prepareFiles.

private void prepareFiles(MetaStore metaStore) throws MetaStoreException {
    List<FileInfo> statusInternals = new ArrayList<>();
    for (int id = 1; id < 6; id++) {
        statusInternals.add(new FileInfo("/file" + id, id + 100, 123L, false, (short) 1, 128 * 1024L, 123123123L, 123123120L, (short) 1, "root", "admin", (byte) 0, (byte) 0));
    }
    metaStore.insertFiles(statusInternals.toArray(new FileInfo[0]));
}

Example 18 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method getPathByFileId.

public String getPathByFileId(long fid) throws MetaStoreException {
    FileInfo fileInfo = metaStore.getFile(fid);
    String path = "";
    if (fileInfo != null) {
        path = fileInfo.getPath();
    }
    return path;
}

Example 19 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFilePlugin method preSubmitCmdletDescriptor.

@Override
public CmdletDescriptor preSubmitCmdletDescriptor(final RuleInfo ruleInfo, TranslateResult tResult, CmdletDescriptor descriptor) {
    for (int i = 0; i < descriptor.getActionSize(); i++) {
        if (COMPACT_ACTION_NAME.equals(descriptor.getActionName(i))) {
            String smallFiles = descriptor.getActionArgs(i).get(HdfsAction.FILE_PATH);
            if (smallFiles != null && !smallFiles.isEmpty()) {
                // Check if small file list is empty
                ArrayList<String> smallFileList = new Gson().fromJson(smallFiles, new TypeToken<ArrayList<String>>() {
                }.getType());
                if (smallFileList == null || smallFileList.isEmpty()) {
                    continue;
                }
                // Get the first small file info
                String firstFile = smallFileList.get(0);
                FileInfo firstFileInfo;
                if (firstFileInfoCache.containsKey(firstFile)) {
                    firstFileInfo = firstFileInfoCache.get(firstFile);
                } else {
                    try {
                        firstFileInfo = metaStore.getFile(firstFile);
                        if (firstFileInfo == null) {
                            LOG.debug("{} is not exist!!!", firstFile);
                            continue;
                        }
                    } catch (MetaStoreException e) {
                        LOG.error(String.format("Failed to get file info of: %s.", firstFile), e);
                        continue;
                    }
                }
                // Get valid compact action arguments
                SmartFilePermission firstFilePermission = new SmartFilePermission(firstFileInfo);
                String firstFileDir = firstFile.substring(0, firstFile.lastIndexOf("/") + 1);
                CompactActionArgs args = getCompactActionArgs(ruleInfo, firstFileDir, firstFilePermission, smallFileList);
                // Set container file path and its permission, file path of this action
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE, args.containerFile);
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(args.containerFilePermission));
                descriptor.addActionArg(i, HdfsAction.FILE_PATH, new Gson().toJson(args.smartFiles));
            }
        }
    }
    return descriptor;
}

Example 20 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFilePlugin method getCompactActionArgs.

/**
 * Get valid compact action arguments.
 */
private CompactActionArgs getCompactActionArgs(RuleInfo ruleInfo, String firstFileDir, SmartFilePermission firstFilePermission, List<String> smallFileList) {
    Map<String, FileInfo> containerFileMap = containerFileInfoCache.get(ruleInfo);
    for (Iterator<Map.Entry<String, FileInfo>> iter = containerFileMap.entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<String, FileInfo> entry = iter.next();
        String containerFilePath = entry.getKey();
        FileInfo containerFileInfo = entry.getValue();
        iter.remove();
        // Get compact action arguments
        String containerFileDir = containerFilePath.substring(0, containerFilePath.lastIndexOf("/") + 1);
        if (firstFileDir.equals(containerFileDir) && firstFilePermission.equals(new SmartFilePermission(containerFileInfo))) {
            List<String> validSmallFiles;
            try {
                validSmallFiles = getValidSmallFiles(containerFileInfo, smallFileList);
            } catch (MetaStoreException e) {
                LOG.error("Failed to get file info of small files.", e);
                continue;
            }
            if (validSmallFiles != null) {
                return new CompactActionArgs(containerFilePath, null, validSmallFiles);
            }
        }
    }
    return genCompactActionArgs(firstFileDir, firstFilePermission, smallFileList);
}