Examples with RunnableTaskInterface com.bonree.brfs.schedulers.task.manager.RunnableTaskInterface used on opensource projects

Example 1 with RunnableTaskInterface

use of com.bonree.brfs.schedulers.task.manager.RunnableTaskInterface in project BRFS by zhangnianli.

the class GatherResourceJob method operation.

@Override
public void operation(JobExecutionContext context) throws Exception {
    if (mountPoints == null) {
        String[] mounts = StringUtils.split(Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_UNMONITOR_PARTITION), ",");
        if (mounts != null) {
            mountPoints = new ArrayList<>(mounts.length);
            for (String mount : mounts) {
                if (BrStringUtils.isEmpty(mount)) {
                    continue;
                }
                mountPoints.add(mount.trim());
            }
        }
    }
    JobDataMap data = context.getJobDetail().getJobDataMap();
    if (data == null || data.isEmpty()) {
        throw new NullPointerException("job data map is empty");
    }
    String dataDir = data.getString(JobDataMapConstract.DATA_PATH);
    String zkPath = data.getString(JobDataMapConstract.BASE_SERVER_ID_PATH);
    String ip = data.getString(JobDataMapConstract.IP);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    CuratorClient client = mcf.getClient();
    if (client == null) {
        LOG.error("zookeeper is empty !!!!");
        return;
    }
    String basePath = zkPath + "/" + mcf.getGroupName();
    String bPath = basePath + "/base/" + mcf.getServerId();
    if (!client.checkExists(bPath)) {
        saveLocal(client, mcf.getServerId(), dataDir, bPath);
    }
    long gatherInveral = data.getLongValueFromString(JobDataMapConstract.GATHER_INVERAL_TIME);
    int count = data.getIntFromString(JobDataMapConstract.CALC_RESOURCE_COUNT);
    StateMetaServerModel metaSource = GatherResource.gatherResource(dataDir, ip, mountPoints);
    if (metaSource != null) {
        queue.add(metaSource);
        LOG.info("gather stat info !!! {}", queue.size());
    }
    int queueSize = queue.size();
    if (queueSize < count) {
        return;
    }
    // 更新任务的可执行资源
    StatServerModel sum = calcStateServer(gatherInveral, dataDir, count);
    if (sum != null) {
        RunnableTaskInterface rt = mcf.getRt();
        rt.update(sum);
    }
    // 计算可用服务
    BaseMetaServerModel base = getClusterBases(client, basePath + "/base", mcf.getGroupName());
    if (base == null) {
        return;
    }
    String serverId = mcf.getServerId();
    // 计算资源值
    ResourceModel resource = GatherResource.calcResourceValue(base, sum, serverId, ip);
    if (resource == null) {
        LOG.warn("calc resource value is null !!!");
        return;
    }
    sendWarnEmail(resource, mcf.getLimitServerResource());
    resource.setServerId(serverId);
    Map<Integer, String> snIds = getStorageNameIdWithName();
    resource.setSnIds(snIds);
    byte[] rdata = JsonUtils.toJsonBytesQuietly(resource);
    String rPath = basePath + "/resource/" + serverId;
    if (!saveDataToZK(client, rPath, rdata)) {
        LOG.error("resource content :{} save to zk fail !!!", JsonUtils.toJsonStringQuietly(resource));
    } else {
        LOG.info("resource: succefull !!!");
    }
    BaseMetaServerModel local = GatherResource.gatherBase(serverId, dataDir, mountPoints);
    if (local == null) {
        LOG.error("gather base data is empty !!!");
        return;
    }
    byte[] bData = JsonUtils.toJsonBytesQuietly(local);
    if (!saveDataToZK(client, bPath, bData)) {
        LOG.error("base content : {} save to zk fail!!!", JsonUtils.toJsonStringQuietly(local));
    }
    saveLocal(client, mcf.getServerId(), dataDir, bPath);
}

Example 2 with RunnableTaskInterface

use of com.bonree.brfs.schedulers.task.manager.RunnableTaskInterface in project BRFS by zhangnianli.

the class OperationTaskJob method operation.

@Override
public void operation(JobExecutionContext context) {
    JobDataMap data = context.getJobDetail().getJobDataMap();
    String dataPath = data.getString(JobDataMapConstract.DATA_PATH);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    if (release == null) {
        throw new NullPointerException("MetaTaskManager is empty !!!");
    }
    List<TaskType> switchList = mcf.getTaskOn();
    if (switchList == null || switchList.isEmpty()) {
        LOG.warn("switch task is empty !!!");
        return;
    }
    SchedulerManagerInterface schd = mcf.getStm();
    if (schd == null) {
        throw new NullPointerException("SchedulerManagerInterface is empty !!!");
    }
    RunnableTaskInterface runTask = mcf.getRt();
    if (runTask == null) {
        throw new NullPointerException("RunnableTaskInterface is empty !!!");
    }
    String typeName;
    String currentTaskName;
    TaskModel task;
    TaskRunPattern runPattern;
    int poolSize;
    int sumbitSize;
    String serverId = mcf.getServerId();
    SumbitTaskInterface sumbitTask;
    // 判断是否有恢复任务，有恢复任务则不进行创建
    boolean rebalanceFlag = WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE);
    for (TaskType taskType : switchList) {
        sumbitTask = null;
        try {
            if (TaskType.SYSTEM_COPY_CHECK.equals(taskType)) {
                continue;
            }
            typeName = taskType.name();
            poolSize = schd.getTaskPoolSize(typeName);
            sumbitSize = schd.getSumbitedTaskCount(typeName);
            // 判断任务是否可以执行
            boolean isRun = runTask.taskRunnable(taskType.code(), poolSize, sumbitSize);
            if (!isRun) {
                LOG.warn("resource is limit !!! skip {} !!!", typeName);
                continue;
            }
            int retryCount = 3;
            if (TaskType.SYSTEM_CHECK.equals(taskType) || TaskType.SYSTEM_MERGER.equals(taskType) || TaskType.SYSTEM_DELETE.equals(taskType)) {
                retryCount = 0;
            }
            Pair<String, TaskModel> taskPair = TaskStateLifeContral.getCurrentOperationTask(release, typeName, serverId, retryCount);
            if (taskPair == null) {
                LOG.warn("taskType :{} taskName: null is vaild ,skiping !!!", typeName);
                continue;
            }
            currentTaskName = taskPair.getFirst();
            task = TaskStateLifeContral.changeRunTaskModel(taskPair.getSecond(), dataPath);
            // 获取执行策略
            runPattern = runTask.taskRunnPattern(task);
            if (runPattern == null) {
                LOG.warn("TaskRunPattern is null will do it once");
                runPattern = new TaskRunPattern();
                runPattern.setRepeateCount(1);
                runPattern.setSleepTime(1000);
            }
            // 创建任务提交信息
            if (TaskType.SYSTEM_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.SYSTEM_CHECK.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemCheckJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.USER_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), UserDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (rebalanceFlag && TaskType.SYSTEM_CHECK.equals(taskType)) {
                LOG.warn("rebalance task running !! Skip {} sumbit", taskType.name());
                continue;
            }
            if (sumbitTask == null) {
                LOG.warn("sumbit type:{}, taskName :{}, taskcontent is null", typeName, currentTaskName);
                continue;
            }
            boolean isSumbit = schd.addTask(typeName, sumbitTask);
            LOG.info("sumbit type:{}, taskName :{}, state:{}", typeName, currentTaskName, isSumbit);
            if (!isSumbit) {
                LOG.warn("next cycle will sumbit against type : {}, taskName : {}", typeName, currentTaskName);
                continue;
            }
            // 更新任务状态
            // 更新任务执行的位置
            data.put(typeName, currentTaskName);
        } catch (Exception e) {
            LOG.error("{}", e);
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel(this.getClass().getSimpleName() + "模块服务发生问题");
            builder.setException(e);
            builder.setMessage("执行任务发生错误");
            builder.setVariable(data.getWrappedMap());
            emailPool.sendEmail(builder);
        }
    }
}

Example 3 with RunnableTaskInterface

use of com.bonree.brfs.schedulers.task.manager.RunnableTaskInterface in project BRFS by zhangnianli.

the class InitTaskManager method initManager.

/**
 * 概述：初始化任务服务系统
 * @throws ParamsErrorException
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void initManager(ResourceTaskConfig managerConfig, ZookeeperPaths zkPath, ServiceManager sm, StorageRegionManager snm, ServerIDManager sim, CuratorClient client) throws Exception {
    managerConfig.printDetail();
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    String serverId = sim.getFirstServerID();
    mcf.setServerId(serverId);
    mcf.setGroupName(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    double diskRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_AVAILABLE_RATE);
    double diskForceRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_AVAILABLE_RATE);
    double diskwriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_WRITE_SPEED);
    double diskForcewriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_WRITE_SPEED);
    long diskRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_REMAIN_SIZE);
    long diskForceRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_REMAIN_SIZE);
    LimitServerResource lmit = new LimitServerResource();
    lmit.setDiskRemainRate(diskRemainRate);
    lmit.setDiskWriteValue(diskwriteValue);
    lmit.setForceDiskRemainRate(diskForceRemainRate);
    lmit.setForceWriteValue(diskForcewriteValue);
    lmit.setRemainWarnSize(diskRemainSize);
    lmit.setRemainForceSize(diskForceRemainSize);
    mcf.setLimitServerResource(lmit);
    // 工厂类添加服务管理
    mcf.setSm(sm);
    // 工厂类添加storageName管理服务
    mcf.setSnm(snm);
    // 1.工厂类添加调度管理
    SchedulerManagerInterface manager = DefaultSchedulersManager.getInstance();
    mcf.setStm(manager);
    // 工厂类添加发布接口
    MetaTaskManagerInterface release = DefaultReleaseTask.getInstance();
    if (release == null) {
        LOG.error("Meta task is empty");
        System.exit(1);
    }
    String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    release.setPropreties(zkAddresses, zkPath.getBaseTaskPath(), zkPath.getBaseLocksPath());
    if (client == null) {
        LOG.error("zk client is empty");
        System.exit(1);
    }
    mcf.setClient(client);
    mcf.setTm(release);
    // 工厂类添加任务可执行接口
    RunnableTaskInterface run = DefaultRunnableTask.getInstance();
    TaskExecutablePattern limit = TaskExecutablePattern.parse(managerConfig);
    run.setLimitParameter(limit);
    mcf.setRt(run);
    mcf.setZkPath(zkPath);
    mcf.setSim(sim);
    // 创建任务线程池
    if (managerConfig.isTaskFrameWorkSwitch()) {
        // 1.创建任务管理服务
        createMetaTaskManager(manager, zkPath, managerConfig, serverId);
        // 2.启动任务线程池
        List<TaskType> tasks = managerConfig.getSwitchOnTaskType();
        if (tasks == null || tasks.isEmpty()) {
            LOG.warn("switch task on  but task type list is empty !!!");
            return;
        }
        createAndStartThreadPool(manager, managerConfig);
        if (tasks.contains(TaskType.SYSTEM_COPY_CHECK)) {
            SumbitTaskInterface copyJob = createCopySimpleTask(managerConfig.getExecuteTaskIntervalTime(), TaskType.SYSTEM_COPY_CHECK.name(), serverId, CopyRecoveryJob.class.getCanonicalName(), zkAddresses, zkPath.getBaseRoutePath(), Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT));
            manager.addTask(TaskType.SYSTEM_COPY_CHECK.name(), copyJob);
        }
        mcf.setTaskOn(tasks);
        // 3.创建执行任务线程池
        createOperationPool(managerConfig, zkPath, tasks, true);
    }
    if (managerConfig.isResourceFrameWorkSwitch()) {
        // 创建资源调度服务
        createResourceManager(manager, zkPath, managerConfig);
    }
}