Examples with BRFSPath com.bonree.brfs.common.utils.BRFSPath used on opensource projects

Example 1 with BRFSPath

use of com.bonree.brfs.common.utils.BRFSPath in project BRFS by zhangnianli.

the class WatchDog method searchPreys.

/**
 * 概述：获取
 * @param sim
 * @param sns
 * @param dataPath
 * @param limitTime
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void searchPreys(ServerIDManager sim, Collection<StorageRegion> sns, String zkHosts, String baseRoutesPath, String dataPath, long limitTime) {
    if (sns == null || sns.isEmpty() || BrStringUtils.isEmpty(dataPath)) {
        LOG.debug("skip search data because is empty");
        return;
    }
    if (isRun) {
        LOG.info("SKip search data because there is one");
        return;
    }
    lastTime = System.currentTimeMillis();
    // sn 目录及文件
    int snId;
    SecondIDParser parser;
    // 初始化zk连接
    if (curatorClient == null) {
        curatorClient = ManagerContralFactory.getInstance().getClient();
    }
    Map<String, String> snMap;
    long granule;
    long snLimitTime;
    for (StorageRegion sn : sns) {
        if (WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE)) {
            LOG.warn("skip search data because there is one reblance");
            return;
        }
        snId = sn.getId();
        granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
        ;
        snLimitTime = limitTime - limitTime % granule;
        LOG.info(" watch dog eat {} :{}", sn.getName(), sn.getId());
        // 单个副本的不做检查
        if (sn.getReplicateNum() <= 1) {
            continue;
        }
        parser = new SecondIDParser(curatorClient, snId, baseRoutesPath);
        // 使用前必须更新路由规则，否则会解析错误
        parser.updateRoute();
        snMap = new HashMap<>();
        snMap.put(BRFSPath.STORAGEREGION, sn.getName());
        List<BRFSPath> sfiles = BRFSFileUtil.scanBRFSFiles(dataPath, snMap, snMap.size(), new BRFSDogFoodsFilter(sim, parser, sn, snLimitTime));
        if (sfiles == null || sfiles.isEmpty()) {
            continue;
        }
        for (BRFSPath brfsPath : sfiles) {
            preys.add(dataPath + FileUtils.FILE_SEPARATOR + brfsPath.toString());
        }
    }
    // 若见采集结果不为空则调用删除线程
    if (preys.size() > 0) {
        isRun = true;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                // 为空跳出
                if (preys == null) {
                    LOG.debug("queue is empty skip !!!");
                    return;
                }
                int count = 0;
                String path;
                while (!preys.isEmpty()) {
                    try {
                        path = preys.poll();
                        boolean deleteFlag = FileUtils.deleteFile(path);
                        LOG.debug("file : {} deleting!", path);
                        if (!deleteFlag) {
                            LOG.info("file : {} cann't delete !!!", path);
                        }
                        count++;
                        if (count % 100 == 0) {
                            Thread.sleep(1000L);
                        }
                    } catch (Exception e) {
                        LOG.error("watch dog delete file error {}", e);
                    }
                }
                isRun = false;
            }
        });
    }
}

Example 2 with BRFSPath

use of com.bonree.brfs.common.utils.BRFSPath in project BRFS by zhangnianli.

the class VirtualRecover method recover.

@Override
public void recover() {
    LOG.info("begin virtual recover");
    // 注册节点
    LOG.info("create:" + selfNode + "-------------" + detail);
    // 无注册的话，则注册，否则不用注册
    while (true) {
        detail = registerNodeDetail(selfNode);
        if (detail != null) {
            LOG.info("register " + selfNode + " is successful!!");
            break;
        }
        LOG.error("register " + selfNode + " is error!!");
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    // 主任务结束，则直接退出
    if (balanceSummary.getTaskStatus().equals(TaskStatus.FINISH)) {
        finishTask();
        return;
    }
    try {
        for (int i = 0; i < delayTime; i++) {
            if (status.get().equals(TaskStatus.CANCEL)) {
                return;
            }
            // 倒计时完毕，则不需要倒计时
            if (!detail.getStatus().equals(ExecutionStatus.INIT)) {
                break;
            }
            if (delayTime - i <= 10) {
                LOG.info("remain time:" + (delayTime - i) + "s, start task!!!");
            } else {
                if ((delayTime - i) % 10 == 0) {
                    LOG.info("remain time:" + (delayTime - i) + "s, start task!!!");
                }
            }
            Thread.sleep(1000);
        }
    } catch (InterruptedException e) {
        LOG.error("task back time count interrupt!!", e);
    }
    detail.setStatus(ExecutionStatus.RECOVER);
    LOG.info("update:" + selfNode + "-------------" + detail);
    updateDetail(selfNode, detail);
    int timeFileCounts = 0;
    String snDataDir = dataDir + FileUtils.FILE_SEPARATOR + storageName;
    if (!FileUtils.isExist(snDataDir)) {
        finishTask();
        return;
    }
    List<String> replicasNames = FileUtils.listFileNames(snDataDir);
    for (String replicasName : replicasNames) {
        String replicasPath = snDataDir + FileUtils.FILE_SEPARATOR + replicasName;
        timeFileCounts += FileUtils.listFileNames(replicasPath).size();
    }
    Thread cosumerThread = new Thread(consumerQueue());
    cosumerThread.start();
    detail.setTotalDirectories(timeFileCounts);
    updateDetail(selfNode, detail);
    String remoteSecondId = balanceSummary.getInputServers().get(0);
    String remoteFirstID = idManager.getOtherFirstID(remoteSecondId, balanceSummary.getStorageIndex());
    String virtualID = balanceSummary.getServerId();
    LOG.info("balance virtual serverId:" + virtualID);
    List<BRFSPath> allPaths = BRFSFileUtil.scanFile(dataDir, storageName);
    for (BRFSPath brfsPath : allPaths) {
        if (status.get().equals(TaskStatus.CANCEL)) {
            break;
        }
        String perFile = dataDir + FileUtils.FILE_SEPARATOR + brfsPath.toString();
        if (!perFile.endsWith(".rd")) {
            String timeFileName = brfsPath.getYear() + FileUtils.FILE_SEPARATOR + brfsPath.getMonth() + FileUtils.FILE_SEPARATOR + brfsPath.getDay() + FileUtils.FILE_SEPARATOR + brfsPath.getHourMinSecond();
            String fileName = brfsPath.getFileName();
            int replicaPot = 0;
            String[] metaArr = fileName.split(NAME_SEPARATOR);
            List<String> fileServerIds = new ArrayList<>();
            for (int j = 1; j < metaArr.length; j++) {
                fileServerIds.add(metaArr[j]);
            }
            if (fileServerIds.contains(virtualID)) {
                // 此处位置需要加1，副本数从1开始
                replicaPot = fileServerIds.indexOf(virtualID) + 1;
                FileRecoverMeta fileMeta = new FileRecoverMeta(perFile, fileName, storageName, timeFileName, Integer.parseInt(brfsPath.getIndex()), replicaPot, remoteFirstID);
                try {
                    fileRecoverQueue.put(fileMeta);
                } catch (InterruptedException e) {
                    LOG.error("put file: " + fileMeta, e);
                }
            }
        }
    }
    // 所有的文件已经处理完毕，等待队列为空
    overFlag = true;
    try {
        cosumerThread.join();
    } catch (InterruptedException e) {
        LOG.error("cosumerThread error!", e);
    }
    finishTask();
}

Example 3 with BRFSPath

use of com.bonree.brfs.common.utils.BRFSPath in project BRFS by zhangnianli.

the class MultiRecover method recover.

@Override
public void recover() {
    LOG.info("begin normal recover");
    // 注册节点
    LOG.info("create:" + selfNode + "-------------" + detail);
    // 无注册的话，则注册，否则不用注册
    while (true) {
        detail = registerNodeDetail(selfNode);
        if (detail != null) {
            LOG.info("register " + selfNode + " is successful!!");
            break;
        }
        LOG.error("register " + selfNode + " is error!!");
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    // 主任务结束，则直接退出
    if (balanceSummary.getTaskStatus().equals(TaskStatus.FINISH)) {
        finishTask();
        return;
    }
    try {
        for (int i = 0; i < delayTime; i++) {
            if (status.get().equals(TaskStatus.CANCEL)) {
                return;
            }
            // 暂时用循环控制，后期重构改成wait notify机制
            while (true) {
                if (!status.get().equals(TaskStatus.PAUSE)) {
                    break;
                }
                LOG.info("sub task is pause for " + balanceSummary);
                Thread.sleep(1000);
            }
            // 倒计时完毕，则不需要倒计时
            if (!detail.getStatus().equals(ExecutionStatus.INIT)) {
                break;
            }
            if (delayTime - i <= 10) {
                LOG.info("remain time:" + (delayTime - i) + "s, start task!!!");
            } else {
                if ((delayTime - i) % 10 == 0) {
                    LOG.info("remain time:" + (delayTime - i) + "s, start task!!!");
                }
            }
            Thread.sleep(1000);
        }
    } catch (InterruptedException e) {
        LOG.error("task back time count interrupt!!", e);
    }
    detail.setStatus(ExecutionStatus.RECOVER);
    LOG.info("update:" + selfNode + "-------------" + detail);
    updateDetail(selfNode, detail);
    String snDataDir = dataDir + FileUtils.FILE_SEPARATOR + storageName;
    if (!FileUtils.isExist(snDataDir)) {
        finishTask();
        return;
    }
    List<BRFSPath> allPaths = BRFSFileUtil.scanFile(dataDir, storageName);
    int fileCounts = allPaths.size();
    // 启动消费队列
    Thread cosumerThread = new Thread(consumerQueue());
    cosumerThread.start();
    detail.setTotalDirectories(fileCounts);
    updateDetail(selfNode, detail);
    LOG.info("deal the local server:" + idManager.getSecondServerID(balanceSummary.getStorageIndex()));
    // dealReplicas(replicasNames, snDataDir);
    for (BRFSPath brfsPath : allPaths) {
        if (status.get().equals(TaskStatus.CANCEL)) {
            return;
        }
        String perFile = dataDir + FileUtils.FILE_SEPARATOR + brfsPath.toString();
        String timeFile = brfsPath.getYear() + FileUtils.FILE_SEPARATOR + brfsPath.getMonth() + FileUtils.FILE_SEPARATOR + brfsPath.getDay() + FileUtils.FILE_SEPARATOR + brfsPath.getHourMinSecond();
        if (!perFile.endsWith(".rd")) {
            dealFile(perFile, brfsPath.getFileName(), timeFile, Integer.parseInt(brfsPath.getIndex()));
        }
    }
    overFlag = true;
    LOG.info("wait cosumer !!");
    try {
        cosumerThread.join();
    } catch (InterruptedException e1) {
        LOG.error("cosumerThread error!", e1);
    }
    finishTask();
}

Example 4 with BRFSPath

use of com.bonree.brfs.common.utils.BRFSPath in project BRFS by zhangnianli.

the class FileWriterManager method rebuildFileWriterbyDir.

public void rebuildFileWriterbyDir(String dataDirPath) {
    Map<String, String> baseMap = new HashMap<>();
    List<BRFSPath> rds = BRFSFileUtil.scanBRFSFiles(dataDirPath, baseMap, baseMap.size(), new BRFSRdFileFilter());
    File rdFile = null;
    File dataFile = null;
    for (BRFSPath path : rds) {
        rdFile = new File(new StringBuilder().append(dataDirPath).append(FileUtils.FILE_SEPARATOR).append(path).toString());
        dataFile = RecordFileBuilder.reverse(rdFile);
        if (!dataFile.exists()) {
            LOG.error("no data file is attached to a existed rd file[{}]!", rdFile.getAbsolutePath());
            rdFile.delete();
            continue;
        }
        try {
            rebuildFileWriter(dataFile);
        } catch (Throwable e) {
            LOG.error("rebuild file[{}] error!", dataFile.getAbsolutePath(), e);
        }
    }
}