Examples with EmailPool com.bonree.brfs.email.EmailPool used on opensource projects

Example 1 with EmailPool

use of com.bonree.brfs.email.EmailPool in project BRFS by zhangnianli.

the class ManagerMetaTaskJob method operation.

@Override
public void operation(JobExecutionContext context) throws Exception {
    LOG.info("revise task work");
    JobDataMap data = context.getJobDetail().getJobDataMap();
    // 任务过期时间 ms
    String ttlTimeStr = data.getString(JobDataMapConstract.TASK_EXPIRED_TIME);
    LOG.info("task ttl time : {}", ttlTimeStr);
    long ttlTime = Long.parseLong(ttlTimeStr);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    // 获取可用服务
    String groupName = mcf.getGroupName();
    ServiceManager sm = mcf.getSm();
    // 2.设置可用服务
    List<String> serverIds = getServerIds(sm, groupName);
    if (serverIds == null || serverIds.isEmpty()) {
        LOG.warn("available server list is null");
        return;
    }
    for (TaskType taskType : TaskType.values()) {
        try {
            release.reviseTaskStat(taskType.name(), ttlTime, serverIds);
        } catch (Exception e) {
            LOG.warn("{}", e.getMessage());
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel(this.getClass().getSimpleName() + "模块服务发生问题");
            builder.setException(e);
            builder.setMessage("管理任务数据发生错误");
            builder.setVariable(data.getWrappedMap());
            emailPool.sendEmail(builder);
        }
    }
    LOG.info("revise task success !!!");
}

Example 2 with EmailPool

use of com.bonree.brfs.email.EmailPool in project BRFS by zhangnianli.

the class CopyCountCheck method collectionSnFiles.

/**
 * 概述：获取集群对应目录的文件
 * @param services
 * @param snList
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static Map<StorageRegion, List<String>> collectionSnFiles(List<Service> services, List<StorageRegion> snList, final Map<String, Long> snTimes) throws Exception {
    Map<StorageRegion, List<String>> snMap = new HashMap<>();
    DiskNodeClient client = null;
    int reCount;
    String snName = null;
    String path;
    List<String> strs;
    long time;
    String dirName;
    String sid;
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    ServerIDManager sim = mcf.getSim();
    CuratorClient zkClient = mcf.getClient();
    SecondIDParser parser;
    String basePath = mcf.getZkPath().getBaseRoutePath();
    int timeout = 10000;
    for (Service service : services) {
        try {
            client = TcpClientUtils.getClient(service.getHost(), service.getPort(), service.getExtraPort(), timeout);
            long granule;
            for (StorageRegion sn : snList) {
                parser = new SecondIDParser(zkClient, sn.getId(), basePath);
                parser.updateRoute();
                sid = sim.getOtherSecondID(service.getServiceId(), sn.getId());
                granule = Duration.parse(sn.getFilePartitionDuration()).toMillis();
                reCount = sn.getReplicateNum();
                snName = sn.getName();
                if (!snTimes.containsKey(snName)) {
                    LOG.debug("sntime don't contain {}", snName);
                    continue;
                }
                time = snTimes.get(snName);
                dirName = TimeUtils.timeInterval(time, granule);
                for (int i = 1; i <= reCount; i++) {
                    path = "/" + snName + "/" + i + "/" + dirName;
                    LOG.debug("path :{}", path);
                    strs = getFileList(parser, client, path, sid);
                    if (strs == null || strs.isEmpty()) {
                        LOG.debug("files is empty {}", path);
                        continue;
                    }
                    LOG.debug("Collection dirName :{},{} size :{}", dirName, path, strs.size());
                    if (!snMap.containsKey(sn)) {
                        snMap.put(sn, new ArrayList<>());
                    }
                    snMap.get(sn).addAll(strs);
                }
            }
        } catch (Exception e) {
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel("collect file execute 模块服务发生问题");
            builder.setException(e);
            builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
            Map<String, String> map = new HashedMap();
            map.put("remote ", service.getHost());
            map.put("connectTimeout", String.valueOf(timeout));
            map.put("sn", StringUtils.isEmpty(snName) ? "" : snName);
            if (snTimes != null && !snTimes.isEmpty()) {
                map.put("sntime", JSON.toJSONString(snTimes));
            }
            builder.setVariable(map);
            emailPool.sendEmail(builder);
            throw e;
        } finally {
            if (client != null) {
                try {
                    client.close();
                } catch (IOException e) {
                    LOG.error("{}", e);
                }
            }
        }
    }
    return clearUnLawFiles(snMap);
}

Example 3 with EmailPool

use of com.bonree.brfs.email.EmailPool in project BRFS by zhangnianli.

the class OperationTaskJob method operation.

@Override
public void operation(JobExecutionContext context) {
    JobDataMap data = context.getJobDetail().getJobDataMap();
    String dataPath = data.getString(JobDataMapConstract.DATA_PATH);
    ManagerContralFactory mcf = ManagerContralFactory.getInstance();
    MetaTaskManagerInterface release = mcf.getTm();
    if (release == null) {
        throw new NullPointerException("MetaTaskManager is empty !!!");
    }
    List<TaskType> switchList = mcf.getTaskOn();
    if (switchList == null || switchList.isEmpty()) {
        LOG.warn("switch task is empty !!!");
        return;
    }
    SchedulerManagerInterface schd = mcf.getStm();
    if (schd == null) {
        throw new NullPointerException("SchedulerManagerInterface is empty !!!");
    }
    RunnableTaskInterface runTask = mcf.getRt();
    if (runTask == null) {
        throw new NullPointerException("RunnableTaskInterface is empty !!!");
    }
    String typeName;
    String currentTaskName;
    TaskModel task;
    TaskRunPattern runPattern;
    int poolSize;
    int sumbitSize;
    String serverId = mcf.getServerId();
    SumbitTaskInterface sumbitTask;
    // 判断是否有恢复任务，有恢复任务则不进行创建
    boolean rebalanceFlag = WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE);
    for (TaskType taskType : switchList) {
        sumbitTask = null;
        try {
            if (TaskType.SYSTEM_COPY_CHECK.equals(taskType)) {
                continue;
            }
            typeName = taskType.name();
            poolSize = schd.getTaskPoolSize(typeName);
            sumbitSize = schd.getSumbitedTaskCount(typeName);
            // 判断任务是否可以执行
            boolean isRun = runTask.taskRunnable(taskType.code(), poolSize, sumbitSize);
            if (!isRun) {
                LOG.warn("resource is limit !!! skip {} !!!", typeName);
                continue;
            }
            int retryCount = 3;
            if (TaskType.SYSTEM_CHECK.equals(taskType) || TaskType.SYSTEM_MERGER.equals(taskType) || TaskType.SYSTEM_DELETE.equals(taskType)) {
                retryCount = 0;
            }
            Pair<String, TaskModel> taskPair = TaskStateLifeContral.getCurrentOperationTask(release, typeName, serverId, retryCount);
            if (taskPair == null) {
                LOG.warn("taskType :{} taskName: null is vaild ,skiping !!!", typeName);
                continue;
            }
            currentTaskName = taskPair.getFirst();
            task = TaskStateLifeContral.changeRunTaskModel(taskPair.getSecond(), dataPath);
            // 获取执行策略
            runPattern = runTask.taskRunnPattern(task);
            if (runPattern == null) {
                LOG.warn("TaskRunPattern is null will do it once");
                runPattern = new TaskRunPattern();
                runPattern.setRepeateCount(1);
                runPattern.setSleepTime(1000);
            }
            // 创建任务提交信息
            if (TaskType.SYSTEM_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.SYSTEM_CHECK.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), SystemCheckJob.class.getCanonicalName(), dataPath);
            }
            if (TaskType.USER_DELETE.equals(taskType)) {
                sumbitTask = createSimpleTask(task, runPattern, currentTaskName, mcf.getServerId(), UserDeleteJob.class.getCanonicalName(), dataPath);
            }
            if (rebalanceFlag && TaskType.SYSTEM_CHECK.equals(taskType)) {
                LOG.warn("rebalance task running !! Skip {} sumbit", taskType.name());
                continue;
            }
            if (sumbitTask == null) {
                LOG.warn("sumbit type:{}, taskName :{}, taskcontent is null", typeName, currentTaskName);
                continue;
            }
            boolean isSumbit = schd.addTask(typeName, sumbitTask);
            LOG.info("sumbit type:{}, taskName :{}, state:{}", typeName, currentTaskName, isSumbit);
            if (!isSumbit) {
                LOG.warn("next cycle will sumbit against type : {}, taskName : {}", typeName, currentTaskName);
                continue;
            }
            // 更新任务状态
            // 更新任务执行的位置
            data.put(typeName, currentTaskName);
        } catch (Exception e) {
            LOG.error("{}", e);
            EmailPool emailPool = EmailPool.getInstance();
            MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
            builder.setModel(this.getClass().getSimpleName() + "模块服务发生问题");
            builder.setException(e);
            builder.setMessage("执行任务发生错误");
            builder.setVariable(data.getWrappedMap());
            emailPool.sendEmail(builder);
        }
    }
}

Example 4 with EmailPool

use of com.bonree.brfs.email.EmailPool in project BRFS by zhangnianli.

the class CopyRecovery method copyFrom.

/**
 * 概述：恢复数据文件
 * @param host 远程主机
 * @param port 端口
 * @param export
 * @param timeout
 * @param remotePath
 * @param localPath
 * @return
 * @user <a href=mailto:zhucg@bonree.com>朱成岗</a>
 */
public static boolean copyFrom(String host, int port, int export, int timeout, String remotePath, String localPath) {
    TcpDiskNodeClient client = null;
    try {
        client = TcpClientUtils.getClient(host, port, export, timeout);
        LOG.debug("{}:{},{}:{}, read {} to local {}", host, port, host, export, remotePath, localPath);
        LocalByteStreamConsumer consumer = new LocalByteStreamConsumer(localPath);
        client.readFile(remotePath, consumer);
        return consumer.getResult().get();
    } catch (InterruptedException | IOException | ExecutionException e) {
        EmailPool emailPool = EmailPool.getInstance();
        MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
        builder.setModel("collect file execute 模块服务发生问题");
        builder.setException(e);
        ManagerContralFactory mcf = ManagerContralFactory.getInstance();
        builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
        Map<String, String> map = new HashedMap();
        map.put("remote ", host);
        map.put("remote path", remotePath);
        map.put("local path", localPath);
        map.put("connectTimeout", String.valueOf(timeout));
        builder.setVariable(map);
        emailPool.sendEmail(builder);
        LOG.error("copy from error {}", e);
        return false;
    } finally {
        if (client != null) {
            try {
                client.closeFile(remotePath);
                client.close();
            } catch (IOException e) {
                LOG.error("close error ", e);
            }
        }
    }
}

Example 5 with EmailPool

use of com.bonree.brfs.email.EmailPool in project BRFS by zhangnianli.

the class CycleJobWithZKTask method execute.

@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
    int taskTypeCode = 0;
    int batchSize = 10;
    String currentTaskName = null;
    int batchIndex = 0;
    TaskType taskType = null;
    String serverId = null;
    JobDataMap data = context.getJobDetail().getJobDataMap();
    boolean isSuccess = true;
    try {
        // 获取当前的任务信息
        taskTypeCode = data.getInt(JobDataMapConstract.TASK_TYPE);
        taskType = TaskType.valueOf(taskTypeCode);
        batchSize = data.getInt(JobDataMapConstract.BATCH_SIZE);
        if (!data.containsKey(JobDataMapConstract.CURRENT_TASK_NAME)) {
            data.put(JobDataMapConstract.CURRENT_TASK_NAME, "");
        }
        serverId = data.getString(JobDataMapConstract.SERVER_ID);
        currentTaskName = data.getString(JobDataMapConstract.CURRENT_TASK_NAME);
        if (!data.containsKey(JobDataMapConstract.CURRENT_INDEX)) {
            data.put(JobDataMapConstract.CURRENT_INDEX, "0");
            LOG.info("task:{}-{} start", taskType.name(), currentTaskName);
        }
        batchIndex = data.getInt(JobDataMapConstract.CURRENT_INDEX);
        LOG.debug("current :{}, batchId : {}", currentTaskName, batchIndex);
        if (batchSize == 0) {
            batchSize = 10;
        }
        if (batchIndex >= 1) {
            operation(context);
        }
    } catch (Exception e) {
        LOG.info("happend Exception :{}", e);
        context.put("ExceptionMessage", e.getMessage());
        caughtException(context);
        isSuccess = false;
        EmailPool emailPool = EmailPool.getInstance();
        MailWorker.Builder builder = MailWorker.newBuilder(emailPool.getProgramInfo());
        builder.setModel(this.getClass().getSimpleName() + "模块服务发生问题");
        builder.setException(e);
        ManagerContralFactory mcf = ManagerContralFactory.getInstance();
        builder.setMessage(mcf.getGroupName() + "(" + mcf.getServerId() + ")服务 执行任务时发生问题");
        builder.setVariable(data.getWrappedMap());
        emailPool.sendEmail(builder);
    } finally {
        // 判断是否有恢复任务，有恢复任务则不进行创建
        if (WatchSomeThingJob.getState(WatchSomeThingJob.RECOVERY_STATUSE)) {
            LOG.warn("rebalance task is running !! skip check copy task");
            return;
        }
        if (batchIndex >= 1) {
            LOG.debug("batch ID :{} {} {} {} {}", batchIndex, taskType, currentTaskName, serverId, isSuccess ? TaskState.RUN : TaskState.EXCEPTION);
            TaskResultModel resultTask = new TaskResultModel();
            resultTask.setSuccess(isSuccess);
            TaskStateLifeContral.updateMapTaskMessage(context, resultTask);
        }
        // 最后一次执行更新任务状态并处理任务
        if (batchIndex == 1) {
            String result = data.getString(JobDataMapConstract.TASK_RESULT);
            TaskResultModel tResult = new TaskResultModel();
            if (!BrStringUtils.isEmpty(result)) {
                tResult = JsonUtils.toObjectQuietly(result, TaskResultModel.class);
            }
            TaskState state = isSuccess && tResult.isSuccess() ? TaskState.FINISH : TaskState.EXCEPTION;
            LOG.info("task:{}-{}-{}  end !", taskType.name(), currentTaskName, state.name());
            LOG.debug("batch ID :{} {} {} {} {}", batchIndex, taskType, currentTaskName, serverId, state);
            TaskStateLifeContral.updateTaskStatusByCompelete(serverId, currentTaskName, taskType.name(), tResult);
            data.put(JobDataMapConstract.CURRENT_INDEX, (batchIndex - 1) + "");
            data.put(JobDataMapConstract.TASK_RESULT, "");
            data.put(JobDataMapConstract.CURRENT_TASK_NAME, "");
            data.put(JobDataMapConstract.CURRENT_INDEX, (batchIndex - 1) + "");
        } else if (batchIndex <= 0) {
            ManagerContralFactory mcf = ManagerContralFactory.getInstance();
            MetaTaskManagerInterface release = mcf.getTm();
            // 创建任务
            createBatchData(release, data, serverId, taskType, batchSize, 3);
        } else {
            // 更新任务状态
            data.put(JobDataMapConstract.CURRENT_INDEX, (batchIndex - 1) + "");
        }
    }
}