Examples with SecondIDParser com.bonree.brfs.rebalance.route.SecondIDParser used on opensource projects

Example 1 with SecondIDParser

use of com.bonree.brfs.rebalance.route.SecondIDParser in project BRFS by zhangnianli.

the class WatchDog method searchPreys.

/**
 * 概述：获取
 * @param sim
 * @param sns
 * @param dataPath
 * @param limitTime
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void searchPreys(ServerIDManager sim, Collection<StorageRegion> sns, String zkHosts, String baseRoutesPath, String dataPath, long limitTime) {
    if (sns == null || sns.isEmpty() || BrStringUtils.isEmpty(dataPath)) {
        LOG.debug("skip search data because is empty");
        return;
    }
    if (isRun) {
        LOG.info("SKip search data because there is one");
        return;
    }
    lastTime = System.currentTimeMillis();
    // sn 目录及文件
    int snId;
    SecondIDParser parser;
    // 初始化zk连接
    if (curatorClient == null) {
        curatorClient = ManagerContralFactory.getInstance().getClient();
    }
    Map<String, String> snMap;
    long granule;
    long snLimitTime;
    for (StorageRegion sn : sns) {
        if (WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE)) {
            LOG.warn("skip search data because there is one reblance");
            return;
        }
        snId = sn.getId();
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        ;
        snLimitTime = limitTime - limitTime % granule;
        LOG.info(" watch dog eat {} :{}", sn.getName(), sn.getId());
        // 单个副本的不做检查
        if (sn.getReplicateNum() <= 1) {
            continue;
        }
        parser = new SecondIDParser(curatorClient, snId, baseRoutesPath);
        // 使用前必须更新路由规则，否则会解析错误
        parser.updateRoute();
        snMap = new HashMap<>();
        snMap.put(BRFSPath.STORAGEREGION, sn.getName());
        List<BRFSPath> sfiles = BRFSFileUtil.scanBRFSFiles(dataPath, snMap, snMap.size(), new BRFSDogFoodsFilter(sim, parser, sn, snLimitTime));
        if (sfiles == null || sfiles.isEmpty()) {
            continue;
        }
        for (BRFSPath brfsPath : sfiles) {
            preys.add(dataPath + FileUtils.FILE_SEPARATOR + brfsPath.toString());
        }
    }
    // 若见采集结果不为空则调用删除线程
    if (preys.size() > 0) {
        isRun = true;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                // 为空跳出
                if (preys == null) {
                    LOG.debug("queue is empty skip !!!");
                    return;
                }
                int count = 0;
                String path;
                while (!preys.isEmpty()) {
                    try {
                        path = preys.poll();
                        boolean deleteFlag = FileUtils.deleteFile(path);
                        LOG.debug("file : {} deleting!", path);
                        if (!deleteFlag) {
                            LOG.info("file : {} cann't delete !!!", path);
                        }
                        count++;
                        if (count % 100 == 0) {
                            Thread.sleep(1000L);
                        }
                    } catch (Exception e) {
                        LOG.error("watch dog delete file error {}", e);
                    }
                }
                isRun = false;
            }
        });
    }
}

Example 2 with SecondIDParser

use of com.bonree.brfs.rebalance.route.SecondIDParser in project BRFS by zhangnianli.

the class CopyCountCheck method collectionSnFiles.

/**
 * 概述：获取集群对应目录的文件
 * @param services
 * @param snList
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, List<String>> collectionSnFiles(List<Service> services, List<StorageRegion> snList, final Map<String, Long> snTimes) throws Exception {
    Map<StorageRegion, List<String>> snMap = new HashMap<>();
    DiskNodeClient client = null;
    int reCount;
    String snName = null;
    String path;
    List<String> strs;
    long time;
    String dirName;
    String sid;
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    CuratorClient zkClient = mcf.getClient();
    SecondIDParser parser;
    String basePath = mcf.getZkPath().getBaseRoutePath();
    int timeout = 10000;
    for (Service service : services) {
        try {
            client = TcpClientUtils.getClient(service.getHost(), service.getPort(), service.getExtraPort(), timeout);
            long granule;
            for (StorageRegion sn : snList) {
                parser = new SecondIDParser(zkClient, sn.getId(), basePath);
                parser.updateRoute();
                sid = sim.getOtherSecondID(service.getServiceId(), sn.getId());
                granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
                reCount = sn.getReplicateNum();
                snName = sn.getName();
                if (!snTimes.containsKey(snName)) {
                    LOG.debug("sntime don't contain {}", snName);
                    continue;
                }
                time = snTimes.get(snName);
                dirName = TimeUtils.timeInterval(time, granule);
                for (int i = 1; i <= reCount; i++) {
                    path = "/" + snName + "/" + i + "/" + dirName;
                    LOG.debug("path :{}", path);
                    strs = getFileList(parser, client, path, sid);
                    if (strs == null || strs.isEmpty()) {
                        LOG.debug("files is empty {}", path);
                        continue;
                    }
                    LOG.debug("Collection dirName :{},{} size :{}", dirName, path, strs.size());
                    if (!snMap.containsKey(sn)) {
                        snMap.put(sn, new ArrayList<>());
                    }
                    snMap.get(sn).addAll(strs);
                }
            }
        } catch (Exception e) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel("collect file execute 模块服务发生问题");
            builder.setException(e);
            builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
            Map<String, String> map = new HashedMap();
            map.put("remote ", service.getHost());
            map.put("connectTimeout", String.valueOf(timeout));
            map.put("sn", StringUtils.isEmpty(snName) ? "" : snName);
            if (snTimes != null && !snTimes.isEmpty()) {
                map.put("sntime", JSON.toJSONString(snTimes));
            }
            builder.setVariable(map);
            emailPool.sendEmail(builder);
            throw e;
        } finally {
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    LOG.error("{}", e);
                }
            }
        }
    }
    return clearUnLawFiles(snMap);
}

Example 3 with SecondIDParser

use of com.bonree.brfs.rebalance.route.SecondIDParser in project BRFS by zhangnianli.

the class CopyRecovery method recoveryDirs.

/**
 * 概述：修复目录
 * @param content
 * @param zkHosts
 * @param baseRoutesPath
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static TaskResultModel recoveryDirs(String content, String zkHosts, String baseRoutesPath, String dataPath) {
    TaskResultModel result = new TaskResultModel();
    BatchAtomModel batch = converStringToBatch(content);
    if (batch == null) {
        result.setSuccess(false);
        LOG.debug("batch is empty");
        return result;
    }
    List<AtomTaskModel> atoms = batch.getAtoms();
    if (atoms == null || atoms.isEmpty()) {
        result.setSuccess(true);
        LOG.debug(" files is empty");
        return result;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    ServiceManager sm = mcf.getSm();
    StorageRegionManager snm = mcf.getSnm();
    CuratorClient curatorClient = mcf.getClient();
    StorageRegion sn;
    SecondIDParser parser;
    String snName;
    int snId;
    AtomTaskResultModel atomR;
    List<String> errors;
    for (AtomTaskModel atom : atoms) {
        atomR = new AtomTaskResultModel();
        atomR.setFiles(atom.getFiles());
        atomR.setSn(atom.getStorageName());
        snName = atom.getStorageName();
        sn = snm.findStorageRegionByName(snName);
        if (sn == null) {
            atomR.setSuccess(false);
            result.setSuccess(false);
            result.add(atomR);
            LOG.debug("sn == null snName :{}", snName);
            continue;
        }
        snId = sn.getId();
        parser = new SecondIDParser(curatorClient, snId, baseRoutesPath);
        parser.updateRoute();
        errors = recoveryFiles(sm, sim, parser, sn, atom, dataPath);
        if (errors == null || errors.isEmpty()) {
            result.add(atomR);
            LOG.debug("result is empty snName:{}", snName);
            continue;
        }
        atomR.addAll(errors);
        atomR.setSuccess(false);
        result.setSuccess(false);
    }
    return result;
}