Search in sources :

Example 46 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class SmallFilePlugin method updateContainerFileInfoCache.

/**
 * Update container file info cache based on containerFileSizeThreshold
 * and cmdlet.
 */
private void updateContainerFileInfoCache(RuleInfo ruleInfo, Map<String, FileInfo> containerFileInfoMap) {
    if (!containerFileInfoMap.isEmpty()) {
        // Remove container file whose size is greater than containerFileSizeThreshold
        for (Map.Entry<String, FileInfo> entry : containerFileInfoMap.entrySet()) {
            if (entry.getValue().getLength() >= containerFileSizeThreshold) {
                containerFileInfoMap.remove(entry.getKey());
            }
        }
        // Remove container file which is being used
        try {
            List<Long> aids = new ArrayList<>();
            List<CmdletInfo> list = cmdletManager.listCmdletsInfo(ruleInfo.getId());
            for (CmdletInfo cmdletInfo : list) {
                if (!CmdletState.isTerminalState(cmdletInfo.getState())) {
                    aids.addAll(cmdletInfo.getAids());
                }
            }
            List<ActionInfo> actionInfos = cmdletManager.getActions(aids);
            for (ActionInfo actionInfo : actionInfos) {
                Map<String, String> args = actionInfo.getArgs();
                if (args.containsKey(SmallFileCompactAction.CONTAINER_FILE)) {
                    containerFileInfoMap.remove(args.get(SmallFileCompactAction.CONTAINER_FILE));
                }
            }
        } catch (IOException e) {
            LOG.error("Failed to get cmdlet and action info.", e);
        }
    }
    containerFileInfoCache.put(ruleInfo, containerFileInfoMap);
}
Also used : ArrayList(java.util.ArrayList) ActionInfo(org.smartdata.model.ActionInfo) IOException(java.io.IOException) FileInfo(org.smartdata.model.FileInfo) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CmdletInfo(org.smartdata.model.CmdletInfo)

Example 47 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class MetaStore method listFileActions.

public List<DetailedFileAction> listFileActions(long rid, int size) throws MetaStoreException {
    if (mapStoragePolicyIdName == null) {
        updateCache();
    }
    List<ActionInfo> actionInfos = getActions(rid, size);
    List<DetailedFileAction> detailedFileActions = new ArrayList<>();
    for (ActionInfo actionInfo : actionInfos) {
        DetailedFileAction detailedFileAction = new DetailedFileAction(actionInfo);
        String filePath = actionInfo.getArgs().get("-file");
        FileInfo fileInfo = getFile(filePath);
        if (fileInfo == null) {
            // LOG.debug("Namespace is not sync! File {} not in file table!", filePath);
            // Add a mock fileInfo
            fileInfo = new FileInfo(filePath, 0L, 0L, false, (short) 0, 0L, 0L, 0L, (short) 0, "root", "root", (byte) 0, (byte) 0);
        }
        detailedFileAction.setFileLength(fileInfo.getLength());
        detailedFileAction.setFilePath(filePath);
        if (actionInfo.getActionName().contains("allssd") || actionInfo.getActionName().contains("onessd") || actionInfo.getActionName().contains("archive") || actionInfo.getActionName().contains("alldisk") || actionInfo.getActionName().contains("onedisk") || actionInfo.getActionName().contains("ramdisk")) {
            detailedFileAction.setTarget(actionInfo.getActionName());
            detailedFileAction.setSrc(mapStoragePolicyIdName.get((int) fileInfo.getStoragePolicy()));
        } else {
            detailedFileAction.setSrc(actionInfo.getArgs().get("-src"));
            detailedFileAction.setTarget(actionInfo.getArgs().get("-dest"));
        }
        detailedFileActions.add(detailedFileAction);
    }
    return detailedFileActions;
}
Also used : FileInfo(org.smartdata.model.FileInfo) DetailedFileAction(org.smartdata.model.DetailedFileAction) ArrayList(java.util.ArrayList) ActionInfo(org.smartdata.model.ActionInfo)

Example 48 with ActionInfo

use of org.smartdata.model.ActionInfo in project SSM by Intel-bigdata.

the class MetaStore method getActions.

public List<ActionInfo> getActions(long rid, long start, long offset) throws MetaStoreException {
    long mark = 0;
    long count = 0;
    List<CmdletInfo> cmdletInfos = cmdletDao.getByRid(rid);
    List<ActionInfo> totalActions = new ArrayList<>();
    for (CmdletInfo cmdletInfo : cmdletInfos) {
        List<Long> aids = cmdletInfo.getAids();
        if (mark + aids.size() >= start + 1) {
            long gap;
            gap = start - mark;
            for (Long aid : aids) {
                if (gap > 0) {
                    gap--;
                    mark++;
                    continue;
                }
                if (count < offset) {
                    ActionInfo actionInfo = getActionById(aid);
                    totalActions.add(actionInfo);
                    count++;
                    mark++;
                } else {
                    return totalActions;
                }
            }
        } else {
            mark += aids.size();
        }
    }
    return totalActions;
}
Also used : ArrayList(java.util.ArrayList) ActionInfo(org.smartdata.model.ActionInfo) CmdletInfo(org.smartdata.model.CmdletInfo)

Aggregations

ActionInfo (org.smartdata.model.ActionInfo)48 Test (org.junit.Test)23 HashMap (java.util.HashMap)16 CmdletInfo (org.smartdata.model.CmdletInfo)16 ArrayList (java.util.ArrayList)12 IOException (java.io.IOException)7 AtomicLong (java.util.concurrent.atomic.AtomicLong)5 SmartAdmin (org.smartdata.admin.SmartAdmin)5 MetaStore (org.smartdata.metastore.MetaStore)5 Path (org.apache.hadoop.fs.Path)4 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)4 MetaStoreException (org.smartdata.metastore.MetaStoreException)4 FileInfo (org.smartdata.model.FileInfo)4 CmdletDescriptor (org.smartdata.model.CmdletDescriptor)3 ActionScheduler (org.smartdata.model.action.ActionScheduler)3 ServiceException (com.google.protobuf.ServiceException)2 ParseException (java.text.ParseException)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 QueueFullException (org.smartdata.exception.QueueFullException)2 DetailedFileAction (org.smartdata.model.DetailedFileAction)2