Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 31 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method getFilesUnderDir.

private List<String> getFilesUnderDir(String dir) throws MetaStoreException {
    dir = dir.endsWith("/") ? dir : dir + "/";
    List<String> fileList = new ArrayList<>();
    List<String> subdirList = new ArrayList<>();
    // get fileInfo in asc order of path to guarantee that
    // the subdir is tackled prior to files or dirs under it
    List<FileInfo> fileInfos = metaStore.getFilesByPrefixInOrder(dir);
    for (FileInfo fileInfo : fileInfos) {
        // just delete subdir instead of deleting all files under it
        if (isUnderDir(fileInfo.getPath(), subdirList)) {
            continue;
        }
        fileList.add(fileInfo.getPath());
        if (fileInfo.isdir()) {
            subdirList.add(fileInfo.getPath());
        }
    }
    return fileList;
}

Example 32 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method generateFileDiff.

private void generateFileDiff(Event.RenameEvent renameEvent) throws MetaStoreException {
    String src = renameEvent.getSrcPath();
    String dest = renameEvent.getDstPath();
    FileInfo info = metaStore.getFile(src);
    // TODO: consider src or dest is ignored by SSM
    if (inBackup(src)) {
        // if not, insert a delete file diff
        if (inBackup(dest)) {
            FileDiff fileDiff = new FileDiff(FileDiffType.RENAME);
            fileDiff.setSrc(src);
            fileDiff.getParameters().put("-dest", dest);
            metaStore.insertFileDiff(fileDiff);
        } else {
            insertDeleteDiff(src, info.isdir());
        }
    } else if (inBackup(dest)) {
        // tackle such case: rename file from outside into backup dir
        if (!info.isdir()) {
            FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
            fileDiff.setSrc(dest);
            fileDiff.getParameters().put("-offset", String.valueOf(0));
            fileDiff.getParameters().put("-length", String.valueOf(info.getLength()));
            metaStore.insertFileDiff(fileDiff);
        } else {
            List<FileInfo> fileInfos = metaStore.getFilesByPrefix(src.endsWith("/") ? src : src + "/");
            for (FileInfo fileInfo : fileInfos) {
                // TODO: cover subdir with no file case
                if (fileInfo.isdir()) {
                    continue;
                }
                FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
                fileDiff.setSrc(fileInfo.getPath().replaceFirst(src, dest));
                fileDiff.getParameters().put("-offset", String.valueOf(0));
                fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
                metaStore.insertFileDiff(fileDiff);
            }
        }
    }
}

Example 33 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method getCreateSql.

// Todo: times and ec policy id, etc.
private String getCreateSql(Event.CreateEvent createEvent) throws IOException, MetaStoreException {
    HdfsFileStatus fileStatus = client.getFileInfo(createEvent.getPath());
    if (fileStatus == null) {
        LOG.debug("Can not get HdfsFileStatus for file " + createEvent.getPath());
        return "";
    }
    FileInfo fileInfo = HadoopUtil.convertFileStatus(fileStatus, createEvent.getPath());
    if (inBackup(fileInfo.getPath())) {
        if (!fileInfo.isdir()) {
            // ignore dir
            FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
            fileDiff.setSrc(fileInfo.getPath());
            fileDiff.getParameters().put("-offset", String.valueOf(0));
            // Note that "-length 0" means create an empty file
            fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
            // TODO add support in CopyFileAction or split into two file diffs
            // add modification_time and access_time to filediff
            fileDiff.getParameters().put("-mtime", "" + fileInfo.getModificationTime());
            // fileDiff.getParameters().put("-atime", "" + fileInfo.getAccessTime());
            // add owner to filediff
            fileDiff.getParameters().put("-owner", "" + fileInfo.getOwner());
            fileDiff.getParameters().put("-group", "" + fileInfo.getGroup());
            // add Permission to filediff
            fileDiff.getParameters().put("-permission", "" + fileInfo.getPermission());
            // add replication count to file diff
            fileDiff.getParameters().put("-replication", "" + fileInfo.getBlockReplication());
            metaStore.insertFileDiff(fileDiff);
        }
    }
    metaStore.deleteFileByPath(fileInfo.getPath());
    metaStore.deleteFileState(fileInfo.getPath());
    metaStore.insertFile(fileInfo);
    return "";
}

Example 34 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class ErasureCodingScheduler method onSchedule.

@Override
public ScheduleResult onSchedule(CmdletInfo cmdletInfo, ActionInfo actionInfo, LaunchCmdlet cmdlet, LaunchAction action, int actionIndex) {
    if (!actions.contains(action.getActionType())) {
        return ScheduleResult.SUCCESS;
    }
    if (actionInfo.getActionName().equals(LIST_EC_ACTION_ID)) {
        return ScheduleResult.SUCCESS;
    }
    String srcPath = actionInfo.getArgs().get(HdfsAction.FILE_PATH);
    if (srcPath == null) {
        actionInfo.appendLog("No file is given in this action!");
        return ScheduleResult.FAIL;
    }
    if (actionInfo.getActionName().equals(CHECK_EC_ACTION_ID)) {
        return ScheduleResult.SUCCESS;
    }
    try {
        // use the default EC policy if an ec action has not been given an EC policy
        if (actionInfo.getActionName().equals(EC_ACTION_ID)) {
            String ecPolicy = actionInfo.getArgs().get(EC_POLICY);
            if (ecPolicy == null || ecPolicy.isEmpty()) {
                String defaultEcPolicy = conf.getTrimmed("dfs.namenode.ec.system.default.policy", "RS-6-3-1024k");
                actionInfo.getArgs().put(EC_POLICY, defaultEcPolicy);
                action.getArgs().put(EC_POLICY, defaultEcPolicy);
            }
        }
        FileInfo fileinfo = metaStore.getFile(srcPath);
        if (fileinfo != null && fileinfo.isdir()) {
            return ScheduleResult.SUCCESS;
        }
        // The below code is just for ec or unec action with file as argument, not directory
        if (isLimitedByThrottle(srcPath)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to schedule {} due to the limitation of throttle!", actionInfo);
            }
            return ScheduleResult.RETRY;
        }
        // For ec or unec, add ecTmp argument
        String tmpName = createTmpName(action);
        action.getArgs().put(EC_TMP, EC_DIR + tmpName);
        actionInfo.getArgs().put(EC_TMP, EC_DIR + tmpName);
    } catch (MetaStoreException ex) {
        LOG.error("Error occurred for getting file info", ex);
        actionInfo.appendLog(ex.getMessage());
        return ScheduleResult.FAIL;
    }
    afterSchedule(actionInfo);
    return ScheduleResult.SUCCESS;
}

Example 35 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method getPathFromInodeDir.

public String getPathFromInodeDir(InodeDirectoryEntry dirEntry) throws MetaStoreException {
    long pid = dirEntry.getParentId();
    String dName = dirEntry.getName();
    FileInfo fileInfo = metaStore.getFile(pid);
    String pDir = "";
    if (fileInfo != null) {
        pDir = formatPath(fileInfo.getPath());
    }
    return pDir.concat(dName);
}