Examples with MetaTaskManagerInterface com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface used on opensource projects

Example 6 with MetaTaskManagerInterface

use of com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface in project BRFS by zhangnianli.

the class CheckCycleJob method operation.

@Override
public void operation(JobExecutionContext context) throws Exception {
    LOG.info("cycle check job work !!!");
    JobDataMap data = context.getJobDetail().getJobDataMap();
    int day = data.getInt(JobDataMapConstract.CHECK_TIME_RANGE);
    if (day <= 0) {
        LOG.warn("skip cycle job!! because check time range is 0");
        return;
    }
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    StorageRegionManager snm = mcf.getSnm();
    ServiceManager sm = mcf.getSm();
    if (WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE)) {
        LOG.warn("rebalance task is running !! skip check copy task ,wait next time to check");
        return;
    }
    List services = sm.getServiceListByGroup(mcf.getGroupName());
    if ((services == null) || (services.isEmpty())) {
        LOG.warn("SKIP create {} task, because service is empty", TaskType.SYSTEM_COPY_CHECK);
        return;
    }
    List<StorageRegion> snList = snm.getStorageRegionList();
    if ((snList == null) || (snList.isEmpty())) {
        LOG.warn("SKIP storagename list is null");
        return;
    }
    long currentTime = System.currentTimeMillis();
    long lGraDay = currentTime - currentTime % 86400000L;
    long sGraDay = lGraDay - day * 86400000L;
    List<StorageRegion> needSns = CopyCountCheck.filterSn(snList, services.size());
    if (needSns == null || needSns.isEmpty()) {
        LOG.warn("no storagename need check copy count ! ");
        return;
    }
    Map<String, List<Long>> snTimes = collectionTimes(needSns, sGraDay, lGraDay);
    if (snTimes == null || snTimes.isEmpty()) {
        LOG.warn("{} - {} time, no data to check copy count", TimeUtils.formatTimeStamp(sGraDay), TimeUtils.formatTimeStamp(lGraDay));
        return;
    }
    List<Map<String, Long>> tTimes = converTimes(snTimes);
    for (Map<String, Long> sourceTimes : tTimes) {
        if (sourceTimes == null || sourceTimes.isEmpty()) {
            continue;
        }
        createSingleTask(release, needSns, services, TaskType.SYSTEM_COPY_CHECK, sourceTimes);
    }
}

Example 7 with MetaTaskManagerInterface

use of com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface in project BRFS by zhangnianli.

the class CreateSystemTaskJob method operation.

@Override
public void operation(JobExecutionContext context) {
    LOG.info("create system task working");
    // 判断是否有恢复任务，有恢复任务则不进行创建
    JobDataMap data = context.getJobDetail().getJobDataMap();
    long checkTtl = data.getLong(JobDataMapConstract.CHECK_TTL);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    // 获取开启的任务名称
    List<TaskType> switchList = mcf.getTaskOn();
    if (switchList == null || switchList.isEmpty()) {
        LOG.warn("switch on task is empty !!!");
        return;
    }
    // 获取可用服务
    String groupName = mcf.getGroupName();
    ServiceManager sm = mcf.getSm();
    // 2.设置可用服务
    List<String> serverIds = CreateSystemTask.getServerIds(sm, groupName);
    if (serverIds == null || serverIds.isEmpty()) {
        LOG.warn("{} available server list is null", groupName);
        return;
    }
    // 3.获取storageName
    StorageRegionManager snm = mcf.getSnm();
    List<StorageRegion> snList = snm.getStorageRegionList();
    if (snList == null || snList.isEmpty()) {
        LOG.info("skip create system task !!! because storageName is null !!!");
        return;
    }
    TaskModel task;
    String taskName;
    TaskTypeModel tmodel;
    long ttl = 0;
    Pair<TaskModel, TaskTypeModel> result;
    List<String> srs = TaskStateLifeContral.getSRs(snm);
    for (TaskType taskType : switchList) {
        if (TaskType.SYSTEM_COPY_CHECK.equals(taskType) || TaskType.USER_DELETE.equals(taskType)) {
            continue;
        }
        TaskStateLifeContral.watchSR(release, srs, taskType.name());
        if (TaskType.SYSTEM_DELETE.equals(taskType)) {
            ttl = 0;
        } else if (TaskType.SYSTEM_CHECK.equals(taskType)) {
            ttl = checkTtl;
        }
        tmodel = release.getTaskTypeInfo(taskType.name());
        if (tmodel == null) {
            tmodel = new TaskTypeModel();
            tmodel.setSwitchFlag(true);
            LOG.warn("taskType{} is switch but metadata is null");
        }
        result = CreateSystemTask.createSystemTask(tmodel, taskType, snList, ttl);
        if (result == null) {
            LOG.warn("create sys task is empty {}", taskType.name());
            continue;
        }
        task = result.getFirst();
        taskName = CreateSystemTask.updateTask(release, task, serverIds, taskType);
        if (!BrStringUtils.isEmpty(taskName)) {
            LOG.info("create {} {} task successfull !!!", taskType.name(), taskName);
            release.setTaskTypeModel(taskType.name(), tmodel);
        }
    }
}

Example 8 with MetaTaskManagerInterface

use of com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface in project BRFS by zhangnianli.

the class InitTaskManager method createOperationPool.

/**
 * 概述：创建任务执行线程池
 * @param confg
 * @param switchList
 * @param isReboot
 * @throws Exception
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
private static void createOperationPool(ResourceTaskConfig confg, ZookeeperPaths zkPath, List<TaskType> switchList, boolean isReboot) throws Exception {
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    SchedulerManagerInterface manager = mcf.getStm();
    MetaTaskManagerInterface release = mcf.getTm();
    String serverId = mcf.getServerId();
    Properties prop = DefaultBaseSchedulers.createSimplePrope(3, 1000);
    boolean createFlag = manager.createTaskPool(TASK_OPERATION_MANAGER, prop);
    if (!createFlag) {
        LOG.error("create task operation error !!!");
        throw new NullPointerException("create task operation error !!!");
    }
    boolean sFlag = manager.startTaskPool(TASK_OPERATION_MANAGER);
    if (!sFlag) {
        LOG.error("create task operation error !!!");
        throw new NullPointerException("start task operation error !!!");
    }
    Map<String, String> dataMap;
    Map<String, String> switchMap = null;
    if (isReboot) {
        // 将任务信息不完全的任务补充完整
        LOG.info("========================================================================================");
        switchMap = recoveryTask(switchList, release, serverId);
        LOG.info("========================================================================================");
    }
    dataMap = JobDataMapConstract.createRebootTaskOpertionDataMap(Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT), switchMap);
    SumbitTaskInterface task = QuartzSimpleInfo.createCycleTaskInfo(TASK_OPERATION_MANAGER, confg.getExecuteTaskIntervalTime(), 60000, dataMap, OperationTaskJob.class);
    boolean sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, task);
    if (sumbitFlag) {
        LOG.info("operation task sumbit complete !!!");
    }
    String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    Map<String, String> watchMap = JobDataMapConstract.createWatchJobMap(zkAddresses);
    SumbitTaskInterface watchJob = QuartzSimpleInfo.createCycleTaskInfo("WATCH_TASK", 5000, -1, watchMap, WatchSomeThingJob.class);
    sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, watchJob);
    if (sumbitFlag) {
        LOG.info("watch task sumbit complete !!!");
    }
    Map<String, String> watchDogMap = JobDataMapConstract.createWatchDogDataMap(zkAddresses, zkPath.getBaseRoutePath(), Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT));
    LOG.info("watch dog map {}", watchDogMap);
    if (watchDogMap == null || watchDogMap.isEmpty()) {
        System.exit(1);
    }
    SumbitTaskInterface watchDogTask = QuartzCronInfo.getInstance("WATCH_DOG", "WATCH_DOG", confg.getWatchDogCron(), watchDogMap, WatchDogJob.class);
    sumbitFlag = manager.addTask(TASK_OPERATION_MANAGER, watchDogTask);
    if (sumbitFlag) {
        LOG.info("watch dog task sumbit complete !!!");
    }
}

Example 9 with MetaTaskManagerInterface

use of com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface in project BRFS by zhangnianli.

the class InitTaskManager method initManager.

/**
 * 概述：初始化任务服务系统
 * @throws ParamsErrorException
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static void initManager(ResourceTaskConfig managerConfig, ZookeeperPaths zkPath, ServiceManager sm, StorageRegionManager snm, ServerIDManager sim, CuratorClient client) throws Exception {
    managerConfig.printDetail();
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    String serverId = sim.getFirstServerID();
    mcf.setServerId(serverId);
    mcf.setGroupName(Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_DATA_SERVICE_GROUP_NAME));
    double diskRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_AVAILABLE_RATE);
    double diskForceRemainRate = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_AVAILABLE_RATE);
    double diskwriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_WRITE_SPEED);
    double diskForcewriteValue = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_WRITE_SPEED);
    long diskRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_DISK_REMAIN_SIZE);
    long diskForceRemainSize = Configs.getConfiguration().GetConfig(ResourceConfigs.CONFIG_LIMIT_FORCE_DISK_REMAIN_SIZE);
    LimitServerResource lmit = new LimitServerResource();
    lmit.setDiskRemainRate(diskRemainRate);
    lmit.setDiskWriteValue(diskwriteValue);
    lmit.setForceDiskRemainRate(diskForceRemainRate);
    lmit.setForceWriteValue(diskForcewriteValue);
    lmit.setRemainWarnSize(diskRemainSize);
    lmit.setRemainForceSize(diskForceRemainSize);
    mcf.setLimitServerResource(lmit);
    // 工厂类添加服务管理
    mcf.setSm(sm);
    // 工厂类添加storageName管理服务
    mcf.setSnm(snm);
    // 1.工厂类添加调度管理
    SchedulerManagerInterface manager = DefaultSchedulersManager.getInstance();
    mcf.setStm(manager);
    // 工厂类添加发布接口
    MetaTaskManagerInterface release = DefaultReleaseTask.getInstance();
    if (release == null) {
        LOG.error("Meta task is empty");
        System.exit(1);
    }
    String zkAddresses = Configs.getConfiguration().GetConfig(CommonConfigs.CONFIG_ZOOKEEPER_ADDRESSES);
    release.setPropreties(zkAddresses, zkPath.getBaseTaskPath(), zkPath.getBaseLocksPath());
    if (client == null) {
        LOG.error("zk client is empty");
        System.exit(1);
    }
    mcf.setClient(client);
    mcf.setTm(release);
    // 工厂类添加任务可执行接口
    RunnableTaskInterface run = DefaultRunnableTask.getInstance();
    TaskExecutablePattern limit = TaskExecutablePattern.parse(managerConfig);
    run.setLimitParameter(limit);
    mcf.setRt(run);
    mcf.setZkPath(zkPath);
    mcf.setSim(sim);
    // 创建任务线程池
    if (managerConfig.isTaskFrameWorkSwitch()) {
        // 1.创建任务管理服务
        createMetaTaskManager(manager, zkPath, managerConfig, serverId);
        // 2.启动任务线程池
        List<TaskType> tasks = managerConfig.getSwitchOnTaskType();
        if (tasks == null || tasks.isEmpty()) {
            LOG.warn("switch task on  but task type list is empty !!!");
            return;
        }
        createAndStartThreadPool(manager, managerConfig);
        if (tasks.contains(TaskType.SYSTEM_COPY_CHECK)) {
            SumbitTaskInterface copyJob = createCopySimpleTask(managerConfig.getExecuteTaskIntervalTime(), TaskType.SYSTEM_COPY_CHECK.name(), serverId, CopyRecoveryJob.class.getCanonicalName(), zkAddresses, zkPath.getBaseRoutePath(), Configs.getConfiguration().GetConfig(DataNodeConfigs.CONFIG_DATA_ROOT));
            manager.addTask(TaskType.SYSTEM_COPY_CHECK.name(), copyJob);
        }
        mcf.setTaskOn(tasks);
        // 3.创建执行任务线程池
        createOperationPool(managerConfig, zkPath, tasks, true);
    }
    if (managerConfig.isResourceFrameWorkSwitch()) {
        // 创建资源调度服务
        createResourceManager(manager, zkPath, managerConfig);
    }
}

Example 10 with MetaTaskManagerInterface

use of com.bonree.brfs.schedulers.task.manager.MetaTaskManagerInterface in project BRFS by zhangnianli.

the class TasksUtils method createCopyTask.

/**
 * 概述：根据CRC校验任务，创建副本数校验任务
 * @param taskName
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static String createCopyTask(String taskName) {
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    List<String> sNames = release.getTaskServerList(TaskType.SYSTEM_CHECK.name(), taskName);
    if (sNames == null || sNames.isEmpty()) {
        LOG.error("CRC task [{}] get serviceList is empty!!!", taskName);
        return null;
    }
    TaskModel task = converyCopyTaskModel(release, taskName);
    return createTask(release, sNames, TaskType.SYSTEM_CHECK.name(), task);
}