Examples with StorageRegion com.bonree.brfs.duplication.storageregion.StorageRegion used on opensource projects

Example 16 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method calcCopyCount.

/**
 * 概述：
 * @param snFiles
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, Pair<List<String>, List<String>>> calcCopyCount(Map<StorageRegion, List<String>> snFiles) {
    if (snFiles == null || snFiles.isEmpty()) {
        return null;
    }
    StorageRegion sn;
    List files;
    Map fileCopyCount;
    Pair<List<String>, List<String>> result;
    Map<StorageRegion, Pair<List<String>, List<String>>> copyMap = new HashMap<StorageRegion, Pair<List<String>, List<String>>>();
    for (Map.Entry<StorageRegion, List<String>> entry : snFiles.entrySet()) {
        sn = entry.getKey();
        files = entry.getValue();
        if (files == null || files.isEmpty()) {
            continue;
        }
        fileCopyCount = calcFileCount(files);
        if (fileCopyCount == null || fileCopyCount.isEmpty()) {
            continue;
        }
        result = filterLoser(fileCopyCount, sn.getReplicateNum());
        if (result == null) {
            continue;
        }
        copyMap.put(sn, result);
    }
    return copyMap;
}

Example 17 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method repairTime.

/**
 * 概述：添加第一次出现的sn
 * @param sourceTimes
 * @param needSns
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<String, Long> repairTime(final Map<String, Long> sourceTimes, List<StorageRegion> needSns) {
    Map<String, Long> repairs = new ConcurrentHashMap<>();
    if (needSns == null || needSns.isEmpty()) {
        return repairs;
    }
    long currentTime = System.currentTimeMillis();
    String snName;
    long startTime;
    long sGra;
    long cGra;
    long granule;
    for (StorageRegion sn : needSns) {
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        cGra = currentTime - currentTime % granule;
        snName = sn.getName();
        if (sourceTimes != null && sourceTimes.containsKey(snName)) {
            startTime = sourceTimes.get(snName);
        } else {
            startTime = sn.getCreateTime();
        }
        if (currentTime - startTime < granule) {
            continue;
        }
        sGra = startTime - startTime % granule;
        if (sGra == cGra) {
            LOG.info("skip {} create copy check task!! because forbid check current time ", snName);
            continue;
        }
        repairs.put(snName, sGra);
    }
    return repairs;
}

Example 18 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CreateSystemTask method creatTaskWithFiles.

/**
 * 概述：创建单个类型任务
 * @param snTimes
 * @param needSn
 * @param taskType
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Pair<TaskModel, Map<String, Long>> creatTaskWithFiles(final Map<String, Long> snTimes, final Map<String, List<String>> snFiles, List<StorageRegion> needSn, TaskType taskType, String taskOperation, long globalTTL) {
    if (needSn == null || snTimes == null) {
        return null;
    }
    String snName;
    long startTime;
    long endTime;
    long currentTime = System.currentTimeMillis();
    List<AtomTaskModel> sumAtoms = new ArrayList<>();
    long ttl;
    Map<String, Long> lastSnTimes = new HashMap<>(snTimes);
    List<String> files = null;
    AtomTaskModel atom;
    long cGraTime;
    long granule = 0;
    for (StorageRegion sn : needSn) {
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        cGraTime = currentTime - (currentTime % granule);
        snName = sn.getName();
        // 获取开始时间
        if (snTimes.containsKey(snName)) {
            startTime = snTimes.get(snName);
        } else {
            continue;
        }
        // 获取有效的过期时间
        ttl = getTTL(sn, taskType, globalTTL);
        endTime = startTime + granule;
        // 当ttl小于等于0 的sn 跳过
        if (ttl < 0) {
            LOG.debug("sn {} don't to create task !!!", snName);
            continue;
        }
        // 当未达到过期的跳过
        if (cGraTime - startTime < ttl || cGraTime - endTime < ttl) {
            LOG.debug("it's not time to check {}", snName);
            continue;
        }
        // 当前粒度不允许操作
        if (cGraTime == startTime) {
            LOG.warn("current time is forbid to check !!!");
            continue;
        }
        // 当无文件不操作
        if (snFiles != null) {
            files = snFiles.get(snName);
        }
        if (files != null && !files.isEmpty()) {
            atom = AtomTaskModel.getInstance(files, snName, taskOperation, sn.getReplicateNum(), startTime, endTime, granule);
            sumAtoms.add(atom);
        }
        lastSnTimes.put(snName, endTime);
    }
    if (sumAtoms == null || sumAtoms.isEmpty()) {
        return new Pair<>(null, lastSnTimes);
    }
    TaskModel task = TaskModel.getInitInstance(taskType, "1", granule);
    task.putAtom(sumAtoms);
    return new Pair<>(task, lastSnTimes);
}

Example 19 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CreateSystemTask method creatSingleTask.

/**
 * 概述：创建单个类型任务
 * @param snTimes
 * @param needSn
 * @param taskType
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Pair<TaskModel, Map<String, Long>> creatSingleTask(final Map<String, Long> snTimes, List<StorageRegion> needSn, TaskType taskType, long globalTTL) {
    String snName;
    long cTime;
    long startTime;
    long endTime;
    long currentTime = System.currentTimeMillis();
    List<AtomTaskModel> sumAtoms = new ArrayList<>();
    long ttl;
    Map<String, Long> lastSnTimes = new HashMap<>(snTimes);
    long cGraTime;
    long granule = 0;
    AtomTaskModel atom;
    for (StorageRegion sn : needSn) {
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        cGraTime = currentTime - currentTime % granule;
        snName = sn.getName();
        cTime = sn.getCreateTime();
        // 获取开始时间
        if (snTimes.containsKey(snName)) {
            startTime = snTimes.get(snName);
        } else {
            startTime = cTime - cTime % granule;
        }
        // 获取有效的过期时间
        ttl = getTTL(sn, taskType, globalTTL);
        endTime = startTime + granule;
        LOG.debug("sn : {} ,ttl:{}, taskType,", sn.getName(), ttl, taskType.name());
        // 当ttl小于等于0 的sn 跳过
        if (ttl <= 0) {
            LOG.debug("sn {} don't to create task !!!", snName);
            continue;
        }
        // 当未达到过期的跳过
        if (cGraTime - startTime < ttl || cGraTime - endTime < ttl) {
            continue;
        }
        // 当前粒度不允许操作
        if (cGraTime == startTime) {
            continue;
        }
        atom = AtomTaskModel.getInstance(null, snName, "", sn.getReplicateNum(), startTime, endTime, granule);
        if (atom == null) {
            continue;
        }
        sumAtoms.add(atom);
        lastSnTimes.put(snName, endTime);
    }
    if (sumAtoms.isEmpty()) {
        return null;
    }
    TaskModel task = TaskModel.getInitInstance(taskType, "", granule);
    task.putAtom(sumAtoms);
    return new Pair<>(task, lastSnTimes);
}

Example 20 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyRecovery method recoveryDirs.

/**
 * 概述：修复目录
 * @param content
 * @param zkHosts
 * @param baseRoutesPath
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static TaskResultModel recoveryDirs(String content, String zkHosts, String baseRoutesPath, String dataPath) {
    TaskResultModel result = new TaskResultModel();
    BatchAtomModel batch = converStringToBatch(content);
    if (batch == null) {
        result.setSuccess(false);
        LOG.debug("batch is empty");
        return result;
    }
    List<AtomTaskModel> atoms = batch.getAtoms();
    if (atoms == null || atoms.isEmpty()) {
        result.setSuccess(true);
        LOG.debug(" files is empty");
        return result;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    ServiceManager sm = mcf.getSm();
    StorageRegionManager snm = mcf.getSnm();
    CuratorClient curatorClient = mcf.getClient();
    StorageRegion sn;
    SecondIDParser parser;
    String snName;
    int snId;
    AtomTaskResultModel atomR;
    List<String> errors;
    for (AtomTaskModel atom : atoms) {
        atomR = new AtomTaskResultModel();
        atomR.setFiles(atom.getFiles());
        atomR.setSn(atom.getStorageName());
        snName = atom.getStorageName();
        sn = snm.findStorageRegionByName(snName);
        if (sn == null) {
            atomR.setSuccess(false);
            result.setSuccess(false);
            result.add(atomR);
            LOG.debug("sn == null snName :{}", snName);
            continue;
        }
        snId = sn.getId();
        parser = new SecondIDParser(curatorClient, snId, baseRoutesPath);
        parser.updateRoute();
        errors = recoveryFiles(sm, sim, parser, sn, atom, dataPath);
        if (errors == null || errors.isEmpty()) {
            result.add(atomR);
            LOG.debug("result is empty snName:{}", snName);
            continue;
        }
        atomR.addAll(errors);
        atomR.setSuccess(false);
        result.setSuccess(false);
    }
    return result;
}