Examples with StorageRegion com.bonree.brfs.duplication.storageregion.StorageRegion used on opensource projects

Example 11 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class GatherResourceJob method calcStateServer.

/**
 * 概述：计算队列的状态信息
 * @param inverTime
 * @param dataDir
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private StatServerModel calcStateServer(long inverTime, String dataDir, int count) {
    StatServerModel sum = null;
    // 0.计算原始状态信息
    List<StatServerModel> lists = GatherResource.calcState(queue, count);
    if (lists == null || lists.isEmpty()) {
        LOG.warn("server state list is null !!");
        return sum;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    // 1-1初始化storagename管理器
    StorageRegionManager snManager = mcf.getSnm();
    // 2.获取storage信息
    List<StorageRegion> storageNames = snManager.getStorageRegionList();
    List<String> storagenameList = getStorageNames(storageNames);
    // 3.计算状态值
    sum = GatherResource.calcStatServerModel(lists, storagenameList, inverTime, dataDir);
    return sum;
}

Example 12 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method collectionSnFiles.

/**
 * 概述：获取集群对应目录的文件
 * @param services
 * @param snList
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, List<String>> collectionSnFiles(List<Service> services, List<StorageRegion> snList, final Map<String, Long> snTimes) throws Exception {
    Map<StorageRegion, List<String>> snMap = new HashMap<>();
    DiskNodeClient client = null;
    int reCount;
    String snName = null;
    String path;
    List<String> strs;
    long time;
    String dirName;
    String sid;
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    CuratorClient zkClient = mcf.getClient();
    SecondIDParser parser;
    String basePath = mcf.getZkPath().getBaseRoutePath();
    int timeout = 10000;
    for (Service service : services) {
        try {
            client = TcpClientUtils.getClient(service.getHost(), service.getPort(), service.getExtraPort(), timeout);
            long granule;
            for (StorageRegion sn : snList) {
                parser = new SecondIDParser(zkClient, sn.getId(), basePath);
                parser.updateRoute();
                sid = sim.getOtherSecondID(service.getServiceId(), sn.getId());
                granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
                reCount = sn.getReplicateNum();
                snName = sn.getName();
                if (!snTimes.containsKey(snName)) {
                    LOG.debug("sntime don't contain {}", snName);
                    continue;
                }
                time = snTimes.get(snName);
                dirName = TimeUtils.timeInterval(time, granule);
                for (int i = 1; i <= reCount; i++) {
                    path = "/" + snName + "/" + i + "/" + dirName;
                    LOG.debug("path :{}", path);
                    strs = getFileList(parser, client, path, sid);
                    if (strs == null || strs.isEmpty()) {
                        LOG.debug("files is empty {}", path);
                        continue;
                    }
                    LOG.debug("Collection dirName :{},{} size :{}", dirName, path, strs.size());
                    if (!snMap.containsKey(sn)) {
                        snMap.put(sn, new ArrayList<>());
                    }
                    snMap.get(sn).addAll(strs);
                }
            }
        } catch (Exception e) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel("collect file execute 模块服务发生问题");
            builder.setException(e);
            builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
            Map<String, String> map = new HashedMap();
            map.put("remote ", service.getHost());
            map.put("connectTimeout", String.valueOf(timeout));
            map.put("sn", StringUtils.isEmpty(snName) ? "" : snName);
            if (snTimes != null && !snTimes.isEmpty()) {
                map.put("sntime", JSON.toJSONString(snTimes));
            }
            builder.setVariable(map);
            emailPool.sendEmail(builder);
            throw e;
        } finally {
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    LOG.error("{}", e);
                }
            }
        }
    }
    return clearUnLawFiles(snMap);
}

Example 13 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method filterSn.

/**
 * 概述：过滤出有效的集合
 * @param sns
 * @param size
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static List<StorageRegion> filterSn(List<StorageRegion> sns, int size) {
    List<StorageRegion> filters = new ArrayList<StorageRegion>();
    if (sns == null || sns.isEmpty()) {
        return filters;
    }
    int count;
    String snName;
    for (StorageRegion sn : sns) {
        count = sn.getReplicateNum();
        snName = sn.getName();
        if (count == 1) {
            LOG.debug("sn {} {} skip", snName, count);
            continue;
        }
        if (count > size) {
            LOG.debug("sn {} {} {} skip", snName, count, size);
            continue;
        }
        filters.add(sn);
    }
    return filters;
}

Example 14 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method lossFiles.

/**
 * 概述：获取缺失副本的
 * @param copyMap
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<String, List<String>> lossFiles(Map<StorageRegion, Pair<List<String>, List<String>>> copyMap) {
    if (copyMap == null || copyMap.isEmpty()) {
        return null;
    }
    Map<String, List<String>> lossMap = new HashMap<String, List<String>>();
    StorageRegion sn;
    String snName;
    Pair<List<String>, List<String>> cache;
    List<String> losss;
    for (Map.Entry<StorageRegion, Pair<List<String>, List<String>>> entry : copyMap.entrySet()) {
        sn = entry.getKey();
        if (sn == null) {
            continue;
        }
        snName = sn.getName();
        if (BrStringUtils.isEmpty(snName)) {
            continue;
        }
        cache = entry.getValue();
        if (cache == null) {
            continue;
        }
        losss = cache.getFirst();
        if (losss == null || losss.isEmpty()) {
            continue;
        }
        lossMap.put(snName, losss);
    }
    return lossMap;
}

Example 15 with StorageRegion

use of com.bonree.brfs.duplication.storageregion.StorageRegion in project BRFS by zhangnianli.

the class CopyCountCheck method clearUnLawFiles.

public static Map<StorageRegion, List<String>> clearUnLawFiles(Map<StorageRegion, List<String>> data) {
    Map<StorageRegion, List<String>> rMap = new HashMap<>();
    if (data == null || data.isEmpty()) {
        return rMap;
    }
    StorageRegion sr;
    List<String> files;
    for (Map.Entry<StorageRegion, List<String>> entry : data.entrySet()) {
        sr = entry.getKey();
        files = entry.getValue();
        files = clearUnLawFiles(files);
        rMap.put(sr, files);
    }
    return rMap;
}