Examples with ManagerContralFactory com.bonree.brfs.schedulers.ManagerContralFactory used on opensource projects

Example 1 with ManagerContralFactory

use of com.bonree.brfs.schedulers.ManagerContralFactory in project BRFS by zhangnianli.

the class TaskStateLifeContral method updateTaskStatusByCompelete.

/**
 * 概述：更新任务状态
 * @param serverId
 * @param taskname
 * @param taskType
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void updateTaskStatusByCompelete(String serverId, String taskname, String taskType, TaskResultModel taskResult) {
    if (BrStringUtils.isEmpty(taskname)) {
        LOG.warn("task name is empty !!! {} {} {}", taskType, taskname, serverId);
        return;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    TaskServerNodeModel sTask = release.getTaskServerContentNodeInfo(taskType, taskname, serverId);
    if (sTask == null) {
        LOG.debug("server task is null !!! {} {} {}", taskType, taskname, serverId);
        sTask = new TaskServerNodeModel();
    }
    LOG.debug("TaskMessage complete  sTask :{}", JsonUtils.toJsonStringQuietly(sTask));
    sTask.setResult(taskResult);
    if (BrStringUtils.isEmpty(sTask.getTaskStartTime())) {
        sTask.setTaskStartTime(TimeUtils.formatTimeStamp(System.currentTimeMillis(), TimeUtils.TIME_MILES_FORMATE));
    }
    sTask.setTaskStopTime(TimeUtils.formatTimeStamp(System.currentTimeMillis(), TimeUtils.TIME_MILES_FORMATE));
    TaskState status = taskResult == null ? TaskState.EXCEPTION : taskResult.isSuccess() ? TaskState.FINISH : TaskState.EXCEPTION;
    sTask.setTaskState(status.code());
    release.updateServerTaskContentNode(serverId, taskname, taskType, sTask);
    LOG.info("Complete server task :{} - {} - {} - {}", taskType, taskname, serverId, TaskState.valueOf(sTask.getTaskState()).name());
    // 更新TaskContent
    List<Pair<String, Integer>> cStatus = release.getServerStatus(taskType, taskname);
    if (cStatus == null || cStatus.isEmpty()) {
        return;
    }
    LOG.debug("complete c List {}", cStatus);
    int cstat;
    boolean isException = false;
    int finishCount = 0;
    int size = cStatus.size();
    for (Pair<String, Integer> pair : cStatus) {
        cstat = pair.getSecond();
        if (TaskState.EXCEPTION.code() == cstat) {
            isException = true;
            finishCount += 1;
        } else if (TaskState.FINISH.code() == cstat) {
            finishCount += 1;
        }
    }
    if (finishCount != size) {
        return;
    }
    TaskModel task = release.getTaskContentNodeInfo(taskType, taskname);
    if (task == null) {
        LOG.debug("task is null !!! {} {} {}", taskType, taskname);
        task = new TaskModel();
        task.setCreateTime(TimeUtils.formatTimeStamp(System.currentTimeMillis(), TimeUtils.TIME_MILES_FORMATE));
    }
    if (isException) {
        task.setTaskState(TaskState.EXCEPTION.code());
    } else {
        task.setTaskState(TaskState.FINISH.code());
    }
    release.updateTaskContentNode(task, taskType, taskname);
    LOG.info("complete task :{} - {} - {}", taskType, taskname, TaskState.valueOf(task.getTaskState()).name());
    if (TaskType.SYSTEM_CHECK.name().equals(taskType) && isException) {
        TaskModel cTask = TasksUtils.converyCopyTaskModel(release, taskname);
        if (cTask == null) {
            LOG.error("[{}]:[{}] task can't recovery !!!", taskType, taskname);
            return;
        }
        String str = TasksUtils.createCopyTask(release, taskname, cTask);
        if (BrStringUtils.isEmpty(str)) {
            boolean flag = release.setTransferTask(taskType, taskname);
            LOG.info("[{}] task [{}] find error ,transfer task create {}  ", taskType, taskname, flag ? "succefull !!!" : " fail !!!");
        } else {
            LOG.info("[{}]:[{}] find error create copy task [{}] to recovery ", taskType, taskname, str);
        }
    }
}

Example 2 with ManagerContralFactory

use of com.bonree.brfs.schedulers.ManagerContralFactory in project BRFS by zhangnianli.

the class CopyCountCheck method collectionSnFiles.

/**
 * 概述：获取集群对应目录的文件
 * @param services
 * @param snList
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, List<String>> collectionSnFiles(List<Service> services, List<StorageRegion> snList, final Map<String, Long> snTimes) throws Exception {
    Map<StorageRegion, List<String>> snMap = new HashMap<>();
    DiskNodeClient client = null;
    int reCount;
    String snName = null;
    String path;
    List<String> strs;
    long time;
    String dirName;
    String sid;
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    CuratorClient zkClient = mcf.getClient();
    SecondIDParser parser;
    String basePath = mcf.getZkPath().getBaseRoutePath();
    int timeout = 10000;
    for (Service service : services) {
        try {
            client = TcpClientUtils.getClient(service.getHost(), service.getPort(), service.getExtraPort(), timeout);
            long granule;
            for (StorageRegion sn : snList) {
                parser = new SecondIDParser(zkClient, sn.getId(), basePath);
                parser.updateRoute();
                sid = sim.getOtherSecondID(service.getServiceId(), sn.getId());
                granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
                reCount = sn.getReplicateNum();
                snName = sn.getName();
                if (!snTimes.containsKey(snName)) {
                    LOG.debug("sntime don't contain {}", snName);
                    continue;
                }
                time = snTimes.get(snName);
                dirName = TimeUtils.timeInterval(time, granule);
                for (int i = 1; i <= reCount; i++) {
                    path = "/" + snName + "/" + i + "/" + dirName;
                    LOG.debug("path :{}", path);
                    strs = getFileList(parser, client, path, sid);
                    if (strs == null || strs.isEmpty()) {
                        LOG.debug("files is empty {}", path);
                        continue;
                    }
                    LOG.debug("Collection dirName :{},{} size :{}", dirName, path, strs.size());
                    if (!snMap.containsKey(sn)) {
                        snMap.put(sn, new ArrayList<>());
                    }
                    snMap.get(sn).addAll(strs);
                }
            }
        } catch (Exception e) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel("collect file execute 模块服务发生问题");
            builder.setException(e);
            builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
            Map<String, String> map = new HashedMap();
            map.put("remote ", service.getHost());
            map.put("connectTimeout", String.valueOf(timeout));
            map.put("sn", StringUtils.isEmpty(snName) ? "" : snName);
            if (snTimes != null && !snTimes.isEmpty()) {
                map.put("sntime", JSON.toJSONString(snTimes));
            }
            builder.setVariable(map);
            emailPool.sendEmail(builder);
            throw e;
        } finally {
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    LOG.error("{}", e);
                }
            }
        }
    }
    return clearUnLawFiles(snMap);
}

Example 3 with ManagerContralFactory

use of com.bonree.brfs.schedulers.ManagerContralFactory in project BRFS by zhangnianli.

the class CopyRecovery method copyFrom.

/**
 * 概述：恢复数据文件
 * @param host 远程主机
 * @param port 端口
 * @param export
 * @param timeout
 * @param remotePath
 * @param localPath
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static boolean copyFrom(String host, int port, int export, int timeout, String remotePath, String localPath) {
    TcpDiskNodeClient client = null;
    try {
        client = TcpClientUtils.getClient(host, port, export, timeout);
        LOG.debug("{}:{},{}:{}, read {} to local {}", host, port, host, export, remotePath, localPath);
        LocalByteStreamConsumer consumer = new LocalByteStreamConsumer(localPath);
        client.readFile(remotePath, consumer);
        return consumer.getResult().get();
    } catch (InterruptedException | IOException | ExecutionException e) {
        EmailPool emailPool = EmailPool.getInstance();
        MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
        builder.setModel("collect file execute 模块服务发生问题");
        builder.setException(e);
        ManagerContralFactory mcf = ManagerContralFactory.getInstance();
        builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
        Map<String, String> map = new HashedMap();
        map.put("remote ", host);
        map.put("remote path", remotePath);
        map.put("local path", localPath);
        map.put("connectTimeout", String.valueOf(timeout));
        builder.setVariable(map);
        emailPool.sendEmail(builder);
        LOG.error("copy from error {}", e);
        return false;
    } finally {
        if (client != null) {
            try {
                client.closeFile(remotePath);
                client.close();
            } catch (IOException e) {
                LOG.error("close error ", e);
            }
        }
    }
}

Example 4 with ManagerContralFactory

use of com.bonree.brfs.schedulers.ManagerContralFactory in project BRFS by zhangnianli.

the class GatherResourceJob method calcStateServer.

/**
 * 概述：计算队列的状态信息
 * @param inverTime
 * @param dataDir
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private StatServerModel calcStateServer(long inverTime, String dataDir, int count) {
    StatServerModel sum = null;
    // 0.计算原始状态信息
    List<StatServerModel> lists = GatherResource.calcState(queue, count);
    if (lists == null || lists.isEmpty()) {
        LOG.warn("server state list is null !!");
        return sum;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    // 1-1初始化storagename管理器
    StorageRegionManager snManager = mcf.getSnm();
    // 2.获取storage信息
    List<StorageRegion> storageNames = snManager.getStorageRegionList();
    List<String> storagenameList = getStorageNames(storageNames);
    // 3.计算状态值
    sum = GatherResource.calcStatServerModel(lists, storagenameList, inverTime, dataDir);
    return sum;
}

Example 5 with ManagerContralFactory

use of com.bonree.brfs.schedulers.ManagerContralFactory in project BRFS by zhangnianli.

the class GatherResourceJob method operation.

@Override
public void operation(JobExecutionContext context) throws Exception {
    if (mountPoints == null) {
        String[] mounts = StringUtils.split(Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_UNMONITOR_PARTITION), ",");
        if (mounts != null) {
            mountPoints = new ArrayList<>(mounts.length);
            for (String mount : mounts) {
                if (BrStringUtils.isEmpty(mount)) {
                    continue;
                }
                mountPoints.add(mount.trim());
            }
        }
    }
    JobDataMap data = context.getJobDetail().getJobDataMap();
    if (data == null || data.isEmpty()) {
        throw new NullPointerException("job data map is empty");
    }
    String dataDir = data.getString(JobDataMapConstract.DATA_PATH);
    String zkPath = data.getString(JobDataMapConstract.BASE_SERVER_ID_PATH);
    String ip = data.getString(JobDataMapConstract.IP);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    CuratorClient client = mcf.getClient();
    if (client == null) {
        LOG.error("zookeeper is empty !!!!");
        return;
    }
    String basePath = zkPath + "/" + mcf.getGroupName();
    String bPath = basePath + "/base/" + mcf.getServerId();
    if (!client.checkExists(bPath)) {
        saveLocal(client, mcf.getServerId(), dataDir, bPath);
    }
    long gatherInveral = data.getLongValueFromString(JobDataMapConstract.GATHER_INVERAL_TIME);
    int count = data.getIntFromString(JobDataMapConstract.CALC_RESOURCE_COUNT);
    StateMetaServerModel metaSource = GatherResource.gatherResource(dataDir, ip, mountPoints);
    if (metaSource != null) {
        queue.add(metaSource);
        LOG.info("gather stat info !!! {}", queue.size());
    }
    int queueSize = queue.size();
    if (queueSize < count) {
        return;
    }
    // 更新任务的可执行资源
    StatServerModel sum = calcStateServer(gatherInveral, dataDir, count);
    if (sum != null) {
        RunnableTaskInterface rt = mcf.getRt();
        rt.update(sum);
    }
    // 计算可用服务
    BaseMetaServerModel base = getClusterBases(client, basePath + "/base", mcf.getGroupName());
    if (base == null) {
        return;
    }
    String serverId = mcf.getServerId();
    // 计算资源值
    ResourceModel resource = GatherResource.calcResourceValue(base, sum, serverId, ip);
    if (resource == null) {
        LOG.warn("calc resource value is null !!!");
        return;
    }
    sendWarnEmail(resource, mcf.getLimitServerResource());
    resource.setServerId(serverId);
    Map<Integer, String> snIds = getStorageNameIdWithName();
    resource.setSnIds(snIds);
    byte[] rdata = JsonUtils.toJsonBytesQuietly(resource);
    String rPath = basePath + "/resource/" + serverId;
    if (!saveDataToZK(client, rPath, rdata)) {
        LOG.error("resource content :{} save to zk fail !!!", JsonUtils.toJsonStringQuietly(resource));
    } else {
        LOG.info("resource: succefull !!!");
    }
    BaseMetaServerModel local = GatherResource.gatherBase(serverId, dataDir, mountPoints);
    if (local == null) {
        LOG.error("gather base data is empty !!!");
        return;
    }
    byte[] bData = JsonUtils.toJsonBytesQuietly(local);
    if (!saveDataToZK(client, bPath, bData)) {
        LOG.error("base content : {} save to zk fail!!!", JsonUtils.toJsonStringQuietly(local));
    }
    saveLocal(client, mcf.getServerId(), dataDir, bPath);
}