Examples with AtomTaskModel com.bonree.brfs.schedulers.task.model.AtomTaskModel used on opensource projects

Example 6 with AtomTaskModel

use of com.bonree.brfs.schedulers.task.model.AtomTaskModel in project BRFS by zhangnianli.

the class CreateSystemTask method creatTaskWithFiles.

/**
 * 概述：创建单个类型任务
 * @param snTimes
 * @param needSn
 * @param taskType
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Pair<TaskModel, Map<String, Long>> creatTaskWithFiles(final Map<String, Long> snTimes, final Map<String, List<String>> snFiles, List<StorageRegion> needSn, TaskType taskType, String taskOperation, long globalTTL) {
    if (needSn == null || snTimes == null) {
        return null;
    }
    String snName;
    long startTime;
    long endTime;
    long currentTime = System.currentTimeMillis();
    List<AtomTaskModel> sumAtoms = new ArrayList<>();
    long ttl;
    Map<String, Long> lastSnTimes = new HashMap<>(snTimes);
    List<String> files = null;
    AtomTaskModel atom;
    long cGraTime;
    long granule = 0;
    for (StorageRegion sn : needSn) {
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        cGraTime = currentTime - (currentTime % granule);
        snName = sn.getName();
        // 获取开始时间
        if (snTimes.containsKey(snName)) {
            startTime = snTimes.get(snName);
        } else {
            continue;
        }
        // 获取有效的过期时间
        ttl = getTTL(sn, taskType, globalTTL);
        endTime = startTime + granule;
        // 当ttl小于等于0 的sn 跳过
        if (ttl < 0) {
            LOG.debug("sn {} don't to create task !!!", snName);
            continue;
        }
        // 当未达到过期的跳过
        if (cGraTime - startTime < ttl || cGraTime - endTime < ttl) {
            LOG.debug("it's not time to check {}", snName);
            continue;
        }
        // 当前粒度不允许操作
        if (cGraTime == startTime) {
            LOG.warn("current time is forbid to check !!!");
            continue;
        }
        // 当无文件不操作
        if (snFiles != null) {
            files = snFiles.get(snName);
        }
        if (files != null && !files.isEmpty()) {
            atom = AtomTaskModel.getInstance(files, snName, taskOperation, sn.getReplicateNum(), startTime, endTime, granule);
            sumAtoms.add(atom);
        }
        lastSnTimes.put(snName, endTime);
    }
    if (sumAtoms == null || sumAtoms.isEmpty()) {
        return new Pair<>(null, lastSnTimes);
    }
    TaskModel task = TaskModel.getInitInstance(taskType, "1", granule);
    task.putAtom(sumAtoms);
    return new Pair<>(task, lastSnTimes);
}

Example 7 with AtomTaskModel

use of com.bonree.brfs.schedulers.task.model.AtomTaskModel in project BRFS by zhangnianli.

the class CreateSystemTask method creatSingleTask.

/**
 * 概述：创建单个类型任务
 * @param snTimes
 * @param needSn
 * @param taskType
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Pair<TaskModel, Map<String, Long>> creatSingleTask(final Map<String, Long> snTimes, List<StorageRegion> needSn, TaskType taskType, long globalTTL) {
    String snName;
    long cTime;
    long startTime;
    long endTime;
    long currentTime = System.currentTimeMillis();
    List<AtomTaskModel> sumAtoms = new ArrayList<>();
    long ttl;
    Map<String, Long> lastSnTimes = new HashMap<>(snTimes);
    long cGraTime;
    long granule = 0;
    AtomTaskModel atom;
    for (StorageRegion sn : needSn) {
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        cGraTime = currentTime - currentTime % granule;
        snName = sn.getName();
        cTime = sn.getCreateTime();
        // 获取开始时间
        if (snTimes.containsKey(snName)) {
            startTime = snTimes.get(snName);
        } else {
            startTime = cTime - cTime % granule;
        }
        // 获取有效的过期时间
        ttl = getTTL(sn, taskType, globalTTL);
        endTime = startTime + granule;
        LOG.debug("sn : {} ,ttl:{}, taskType,", sn.getName(), ttl, taskType.name());
        // 当ttl小于等于0 的sn 跳过
        if (ttl <= 0) {
            LOG.debug("sn {} don't to create task !!!", snName);
            continue;
        }
        // 当未达到过期的跳过
        if (cGraTime - startTime < ttl || cGraTime - endTime < ttl) {
            continue;
        }
        // 当前粒度不允许操作
        if (cGraTime == startTime) {
            continue;
        }
        atom = AtomTaskModel.getInstance(null, snName, "", sn.getReplicateNum(), startTime, endTime, granule);
        if (atom == null) {
            continue;
        }
        sumAtoms.add(atom);
        lastSnTimes.put(snName, endTime);
    }
    if (sumAtoms.isEmpty()) {
        return null;
    }
    TaskModel task = TaskModel.getInitInstance(taskType, "", granule);
    task.putAtom(sumAtoms);
    return new Pair<>(task, lastSnTimes);
}

Example 8 with AtomTaskModel

use of com.bonree.brfs.schedulers.task.model.AtomTaskModel in project BRFS by zhangnianli.

the class TasksUtils method collectAtoms.

/**
 * 概述：收集子任务
 * @param sMap
 * @param count
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static List<AtomTaskModel> collectAtoms(Map<String, AtomTaskModel> sMap, int count) {
    if (sMap == null || sMap.isEmpty() || count <= 1) {
        return null;
    }
    List<String> eFiles;
    List<AtomTaskModel> atoms = new ArrayList<>();
    for (AtomTaskModel atom : sMap.values()) {
        eFiles = atom.getFiles();
        if (eFiles == null || eFiles.isEmpty()) {
            continue;
        }
        eFiles = collectFiles(eFiles, count);
        if (eFiles == null || eFiles.isEmpty()) {
            continue;
        }
        atom.getFiles().clear();
        atom.setFiles(eFiles);
        atoms.add(atom);
    }
    return atoms;
}

Example 9 with AtomTaskModel

use of com.bonree.brfs.schedulers.task.model.AtomTaskModel in project BRFS by zhangnianli.

the class TasksUtils method getErrorFile.

/**
 * 概述：生成任务信息
 * @param taskContents
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static TaskModel getErrorFile(List<TaskServerNodeModel> taskContents) {
    if (taskContents == null || taskContents.isEmpty()) {
        return null;
    }
    TaskResultModel tmpR;
    List<AtomTaskResultModel> tmpRs;
    Map<String, Map<String, AtomTaskModel>> rmap = new HashMap<>();
    Map<String, AtomTaskModel> emap;
    String snName;
    String key;
    AtomTaskModel atom;
    Map<String, Integer> snMap = new HashMap<>();
    for (TaskServerNodeModel serverModel : taskContents) {
        tmpR = serverModel.getResult();
        if (tmpR == null) {
            continue;
        }
        tmpRs = tmpR.getAtoms();
        if (tmpRs == null || tmpRs.isEmpty()) {
            continue;
        }
        for (AtomTaskResultModel r : tmpRs) {
            if (r == null) {
                continue;
            }
            snName = r.getSn();
            if (!rmap.containsKey(snName)) {
                rmap.put(snName, new HashMap<>());
            }
            if (!snMap.containsKey(snName)) {
                snMap.put(snName, r.getPartNum());
            }
            emap = rmap.get(snName);
            key = r.getDataStartTime() + "_" + r.getDataStopTime();
            long granule = TimeUtils.getMiles(r.getDataStopTime(), TimeUtils.TIME_MILES_FORMATE) - TimeUtils.getMiles(r.getDataStartTime(), TimeUtils.TIME_MILES_FORMATE);
            if (!emap.containsKey(key)) {
                atom = AtomTaskModel.getInstance(null, snName, CopyCheckJob.RECOVERY_CRC, r.getPartNum(), r.getDataStartTime(), r.getDataStopTime(), granule);
                emap.put(key, atom);
            }
            atom = emap.get(key);
            atom.addAllFiles(r.getFiles());
        }
    }
    List<AtomTaskModel> tList = filterError(rmap, snMap);
    if (tList == null || tList.isEmpty()) {
        return null;
    }
    TaskModel tTask = new TaskModel();
    tTask.setCreateTime(TimeUtils.formatTimeStamp(System.currentTimeMillis(), TimeUtils.TIME_MILES_FORMATE));
    tTask.setAtomList(tList);
    tTask.setTaskState(TaskState.INIT.code());
    tTask.setTaskType(TaskType.SYSTEM_COPY_CHECK.code());
    return tTask;
}

Example 10 with AtomTaskModel

use of com.bonree.brfs.schedulers.task.model.AtomTaskModel in project BRFS by zhangnianli.

the class CopyRecovery method recoveryDirs.

/**
 * 概述：修复目录
 * @param content
 * @param zkHosts
 * @param baseRoutesPath
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static TaskResultModel recoveryDirs(String content, String zkHosts, String baseRoutesPath, String dataPath) {
    TaskResultModel result = new TaskResultModel();
    BatchAtomModel batch = converStringToBatch(content);
    if (batch == null) {
        result.setSuccess(false);
        LOG.debug("batch is empty");
        return result;
    }
    List<AtomTaskModel> atoms = batch.getAtoms();
    if (atoms == null || atoms.isEmpty()) {
        result.setSuccess(true);
        LOG.debug(" files is empty");
        return result;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    ServiceManager sm = mcf.getSm();
    StorageRegionManager snm = mcf.getSnm();
    CuratorClient curatorClient = mcf.getClient();
    StorageRegion sn;
    SecondIDParser parser;
    String snName;
    int snId;
    AtomTaskResultModel atomR;
    List<String> errors;
    for (AtomTaskModel atom : atoms) {
        atomR = new AtomTaskResultModel();
        atomR.setFiles(atom.getFiles());
        atomR.setSn(atom.getStorageName());
        snName = atom.getStorageName();
        sn = snm.findStorageRegionByName(snName);
        if (sn == null) {
            atomR.setSuccess(false);
            result.setSuccess(false);
            result.add(atomR);
            LOG.debug("sn == null snName :{}", snName);
            continue;
        }
        snId = sn.getId();
        parser = new SecondIDParser(curatorClient, snId, baseRoutesPath);
        parser.updateRoute();
        errors = recoveryFiles(sm, sim, parser, sn, atom, dataPath);
        if (errors == null || errors.isEmpty()) {
            result.add(atomR);
            LOG.debug("result is empty snName:{}", snName);
            continue;
        }
        atomR.addAll(errors);
        atomR.setSuccess(false);
        result.setSuccess(false);
    }
    return result;
}