Examples with SumbitTaskInterface com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface used on opensource projects

Example 1 with SumbitTaskInterface

use of com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface in project BRFS by zhangnianli.

the class MetaTaskLeaderManager method sumbitTask.

/**
 * 概述：提交任务
 * @throws ParamsErrorException
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private void sumbitTask() throws ParamsErrorException {
    Map<String, String> createDataMap = JobDataMapConstract.createCreateDataMap(config);
    SumbitTaskInterface createJob = QuartzSimpleInfo.createCycleTaskInfo("CREATE_SYSTEM_TASK", config.getCreateTaskIntervalTime(), 60000, createDataMap, CreateSystemTaskJob.class);
    Map<String, String> metaDataMap = JobDataMapConstract.createMetaDataMap(config);
    SumbitTaskInterface metaJob = QuartzSimpleInfo.createCycleTaskInfo("META_MANAGER_TASK", config.getCreateTaskIntervalTime(), 60000, metaDataMap, ManagerMetaTaskJob.class);
    Map<String, String> copyJobMap = JobDataMapConstract.createCopyCheckMap(config);
    SumbitTaskInterface checkJob = QuartzSimpleInfo.createCycleTaskInfo("COPY_CHECK_TASK", config.getCreateCheckJobTaskervalTime(), 60000, copyJobMap, CopyCheckJob.class);
    boolean isSuccess = false;
    isSuccess = this.manager.addTask(META_TASK_MANAGER, createJob);
    LOG.info("sumbit create Job {} ", isSuccess ? " Sucess" : "Fail");
    isSuccess = this.manager.addTask(META_TASK_MANAGER, metaJob);
    LOG.info("sumbit meta Job {} ", isSuccess ? " Sucess" : "Fail");
    boolean createFlag = config.getTaskPoolSwitchMap().get(TaskType.SYSTEM_COPY_CHECK.name());
    if (createFlag) {
        isSuccess = this.manager.addTask(META_TASK_MANAGER, checkJob);
        LOG.info("sumbit Create Check Job {} ", isSuccess ? " Sucess" : "Fail");
    } else {
        LOG.info("=======> create copy check thread fail");
    }
}

Example 2 with SumbitTaskInterface

use of com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface in project BRFS by zhangnianli.

the class OperationTaskJob method operation.

@Override
public void operation(JobExecutionContext context) {
    JobDataMap data = context.getJobDetail().getJobDataMap();
    String dataPath = data.getString(JobDataMapConstract.DATA_PATH);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    if (release == null) {
        throw new NullPointerException("MetaTaskManager is empty !!!");
    }
    List<TaskType> switchList = mcf.getTaskOn();
    if (switchList == null || switchList.isEmpty()) {
        LOG.warn("switch task is empty !!!");
        return;
    }
    SchedulerManagerInterface schd = mcf.getStm();
    if (schd == null) {
        throw new NullPointerException("SchedulerManagerInterface is empty !!!");
    }
    RunnableTaskInterface runTask = mcf.getRt();
    if (runTask == null) {
        throw new NullPointerException("RunnableTaskInterface is empty !!!");
    }
    String typeName;
    String currentTaskName;
    TaskModel task;
    TaskRunPattern runPattern;
    int poolSize;
    int sumbitSize;
    String serverId = mcf.getServerId();
    SumbitTaskInterface sumbitTask;
    // 判断是否有恢复任务，有恢复任务则不进行创建
    boolean rebalanceFlag = WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE);
    for (TaskType taskType : switchList) {
        sumbitTask = null;
        try {
            if (TaskType.SYSTEM_COPY_CHECK.equals(taskType)) {
                continue;
            }
            typeName = taskType.name();
            poolSize = schd.getTaskPoolSize(typeName);
            sumbitSize = schd.getSumbitedTaskCount(typeName);
            // 判断任务是否可以执行
            boolean isRun = runTask.taskRunnable(taskType.code(), poolSize, sumbitSize);
            if (!isRun) {
                LOG.warn("resource is limit !!! skip {} !!!", typeName);
                continue;
            }
            int retryCount = 3;
            if (TaskType.SYSTEM_CHECK.equals(taskType) || TaskType.SYSTEM_MERGER.equals(taskType) || TaskType.SYSTEM_DELETE.equals(taskType)) {
                retryCount = 0;
            }
            Pair<String, TaskModel> taskPair = TaskStateLifeContral.getCurrentOperationTask(release, typeName, serverId, retryCount);
            if (taskPair == null) {
                LOG.warn("taskType :{} taskName: null is vaild ,skiping !!!", typeName);
                continue;
            }
            currentTaskName = taskPair.getFirst();
            task = TaskStateLifeContral.changeRunTaskModel(taskPair.getSecond(), dataPath);
            // 获取执行策略
            runPattern = runTask.taskRunnPattern(task);
            if (runPattern == null) {
                LOG.warn("TaskRunPattern is null will do it once");
                runPattern = new TaskRunPattern();
                runPattern.setRepeateCount(1);
                runPattern.setSleepTime(1000);
            }
            // 创建任务提交信息
            if (TaskType.SYSTEM_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.SYSTEM_CHECK.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemCheckJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.USER_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), UserDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (rebalanceFlag && TaskType.SYSTEM_CHECK.equals(taskType)) {
                LOG.warn("rebalance task running !! Skip {} sumbit", taskType.name());
                continue;
            }
            if (sumbitTask == null) {
                LOG.warn("sumbit type:{}, taskName :{}, taskcontent is null", typeName, currentTaskName);
                continue;
            }
            boolean isSumbit = schd.addTask(typeName, sumbitTask);
            LOG.info("sumbit type:{}, taskName :{}, state:{}", typeName, currentTaskName, isSumbit);
            if (!isSumbit) {
                LOG.warn("next cycle will sumbit against type : {}, taskName : {}", typeName, currentTaskName);
                continue;
            }
            // 更新任务状态
            // 更新任务执行的位置
            data.put(typeName, currentTaskName);
        } catch (Exception e) {
            LOG.error("{}", e);
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel(this.getClass().getSimpleName() + "模块服务发生问题");
            builder.setException(e);
            builder.setMessage("执行任务发生错误");
            builder.setVariable(data.getWrappedMap());
            emailPool.sendEmail(builder);
        }
    }
}

Example 3 with SumbitTaskInterface

use of com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface in project BRFS by zhangnianli.

the class InitTaskManager method createResourceManager.

/**
 * 概述：创建资源管理
 * @param manager
 * @param zkPaths
 * @param config
 * @throws Exception
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private static void createResourceManager(SchedulerManagerInterface manager, ZookeeperPaths zkPaths, ResourceTaskConfig config) throws Exception {
    // 1.引入第三方lib库，资源采集时需要用到
    LibUtils.loadLibraryPath(config.getLibPath());
    // 2.采集基本信息上传到 zk
    String serverId = ManagerContralFactory.getInstance().getServerId();
    String zkAddress = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    // 3.创建资源采集线程池
    Properties prop = DefaultBaseSchedulers.createSimplePrope(2, 1000);
    manager.createTaskPool(RESOURCE_MANAGER, prop);
    boolean cFlag = manager.startTaskPool(RESOURCE_MANAGER);
    if (!cFlag) {
        LOG.error("{} start fail !!!", RESOURCE_MANAGER);
    }
    // 4.创建采集任务信息
    Map<String, String> gatherMap = JobDataMapConstract.createGatherResourceDataMap(config, serverId, zkPaths.getBaseResourcesPath(), zkAddress);
    SumbitTaskInterface gatherInterface = QuartzSimpleInfo.createCycleTaskInfo(GatherResourceJob.class.getSimpleName(), config.getGatherResourceInveralTime(), 2000, gatherMap, GatherResourceJob.class);
    boolean taskFlag = manager.addTask(RESOURCE_MANAGER, gatherInterface);
    if (!taskFlag) {
        LOG.error("sumbit gather job fail !!!");
    }
    LOG.info("GATHER successful !!!");
// 2.创建同步信息
// Map<String,String> syncMap = JobDataMapConstract.createAsynResourceDataMap(config);
// SumbitTaskInterface syncInterface = QuartzSimpleInfo.createCycleTaskInfo(AsynJob.class.getSimpleName(), config.getGatherResourceInveralTime(), 2000, syncMap, AsynJob.class);
// taskFlag = manager.addTask(RESOURCE_MANAGER, syncInterface);
// if(!taskFlag){
// LOG.error("sumbit asyn job fail !!!");
// }
}

Example 4 with SumbitTaskInterface

use of com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface in project BRFS by zhangnianli.

the class InitTaskManager method createOperationPool.

/**
 * 概述：创建任务执行线程池
 * @param confg
 * @param switchList
 * @param isReboot
 * @throws Exception
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private static void createOperationPool(ResourceTaskConfig confg, ZookeeperPaths zkPath, List<TaskType> switchList, boolean isReboot) throws Exception {
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    SchedulerManagerInterface manager = mcf.getStm();
    MetaTaskManagerInterface release = mcf.getTm();
    String serverId = mcf.getServerId();
    Properties prop = DefaultBaseSchedulers.createSimplePrope(3, 1000);
    boolean createFlag = manager.createTaskPool(TASK_OPERATION_MANAGER, prop);
    if (!createFlag) {
        LOG.error("create task operation error !!!");
        throw new NullPointerException("create task operation error !!!");
    }
    boolean sFlag = manager.startTaskPool(TASK_OPERATION_MANAGER);
    if (!sFlag) {
        LOG.error("create task operation error !!!");
        throw new NullPointerException("start task operation error !!!");
    }
    Map<String, String> dataMap;
    Map<String, String> switchMap = null;
    if (isReboot) {
        // 将任务信息不完全的任务补充完整
        LOG.info("========================================================================================");
        switchMap = recoveryTask(switchList, release, serverId);
        LOG.info("========================================================================================");
    }
    dataMap = JobDataMapConstract.createRebootTaskOpertionDataMap(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT), switchMap);
    SumbitTaskInterface task = QuartzSimpleInfo.createCycleTaskInfo(TASK_OPERATION_MANAGER, confg.getExecuteTaskIntervalTime(), 60000, dataMap, OperationTaskJob.class);
    boolean sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, task);
    if (sumbitFlag) {
        LOG.info("operation task sumbit complete !!!");
    }
    String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    Map<String, String> watchMap = JobDataMapConstract.createWatchJobMap(zkAddresses);
    SumbitTaskInterface watchJob = QuartzSimpleInfo.createCycleTaskInfo("WATCH_TASK", 5000, -1, watchMap, WatchSomeThingJob.class);
    sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, watchJob);
    if (sumbitFlag) {
        LOG.info("watch task sumbit complete !!!");
    }
    Map<String, String> watchDogMap = JobDataMapConstract.createWatchDogDataMap(zkAddresses, zkPath.getBaseRoutePath(), Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT));
    LOG.info("watch dog map {}", watchDogMap);
    if (watchDogMap == null || watchDogMap.isEmpty()) {
        System.exit(1);
    }
    SumbitTaskInterface watchDogTask = QuartzCronInfo.getInstance("WATCH_DOG", "WATCH_DOG", confg.getWatchDogCron(), watchDogMap, WatchDogJob.class);
    sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, watchDogTask);
    if (sumbitFlag) {
        LOG.info("watch dog task sumbit complete !!!");
    }
}

Example 5 with SumbitTaskInterface

use of com.bonree.brfs.schedulers.task.meta.SumbitTaskInterface in project BRFS by zhangnianli.

the class InitTaskManager method initManager.

/**
 * 概述：初始化任务服务系统
 * @throws ParamsErrorException
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void initManager(ResourceTaskConfig managerConfig, ZookeeperPaths zkPath, ServiceManager sm, StorageRegionManager snm, ServerIDManager sim, CuratorClient client) throws Exception {
    managerConfig.printDetail();
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    String serverId = sim.getFirstServerID();
    mcf.setServerId(serverId);
    mcf.setGroupName(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    double diskRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_AVAILABLE_RATE);
    double diskForceRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_AVAILABLE_RATE);
    double diskwriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_WRITE_SPEED);
    double diskForcewriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_WRITE_SPEED);
    long diskRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_REMAIN_SIZE);
    long diskForceRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_REMAIN_SIZE);
    LimitServerResource lmit = new LimitServerResource();
    lmit.setDiskRemainRate(diskRemainRate);
    lmit.setDiskWriteValue(diskwriteValue);
    lmit.setForceDiskRemainRate(diskForceRemainRate);
    lmit.setForceWriteValue(diskForcewriteValue);
    lmit.setRemainWarnSize(diskRemainSize);
    lmit.setRemainForceSize(diskForceRemainSize);
    mcf.setLimitServerResource(lmit);
    // 工厂类添加服务管理
    mcf.setSm(sm);
    // 工厂类添加storageName管理服务
    mcf.setSnm(snm);
    // 1.工厂类添加调度管理
    SchedulerManagerInterface manager = DefaultSchedulersManager.getInstance();
    mcf.setStm(manager);
    // 工厂类添加发布接口
    MetaTaskManagerInterface release = DefaultReleaseTask.getInstance();
    if (release == null) {
        LOG.error("Meta task is empty");
        System.exit(1);
    }
    String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    release.setPropreties(zkAddresses, zkPath.getBaseTaskPath(), zkPath.getBaseLocksPath());
    if (client == null) {
        LOG.error("zk client is empty");
        System.exit(1);
    }
    mcf.setClient(client);
    mcf.setTm(release);
    // 工厂类添加任务可执行接口
    RunnableTaskInterface run = DefaultRunnableTask.getInstance();
    TaskExecutablePattern limit = TaskExecutablePattern.parse(managerConfig);
    run.setLimitParameter(limit);
    mcf.setRt(run);
    mcf.setZkPath(zkPath);
    mcf.setSim(sim);
    // 创建任务线程池
    if (managerConfig.isTaskFrameWorkSwitch()) {
        // 1.创建任务管理服务
        createMetaTaskManager(manager, zkPath, managerConfig, serverId);
        // 2.启动任务线程池
        List<TaskType> tasks = managerConfig.getSwitchOnTaskType();
        if (tasks == null || tasks.isEmpty()) {
            LOG.warn("switch task on  but task type list is empty !!!");
            return;
        }
        createAndStartThreadPool(manager, managerConfig);
        if (tasks.contains(TaskType.SYSTEM_COPY_CHECK)) {
            SumbitTaskInterface copyJob = createCopySimpleTask(managerConfig.getExecuteTaskIntervalTime(), TaskType.SYSTEM_COPY_CHECK.name(), serverId, CopyRecoveryJob.class.getCanonicalName(), zkAddresses, zkPath.getBaseRoutePath(), Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT));
            manager.addTask(TaskType.SYSTEM_COPY_CHECK.name(), copyJob);
        }
        mcf.setTaskOn(tasks);
        // 3.创建执行任务线程池
        createOperationPool(managerConfig, zkPath, tasks, true);
    }
    if (managerConfig.isResourceFrameWorkSwitch()) {
        // 创建资源调度服务
        createResourceManager(manager, zkPath, managerConfig);
    }
}