Examples with JobConfig com.vip.saturn.job.console.domain.JobConfig used on opensource projects

Example 46 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ExecutorController method convertJobConfig.

private JobConfig convertJobConfig(int sheetNumber, int rowNumber, Cell[] rowCells) throws SaturnJobConsoleException {
    JobConfig jobConfig = new JobConfig();
    String jobName = getContents(rowCells, 0);
    if (jobName == null || jobName.trim().isEmpty()) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 1, "作业名必填。"));
    }
    if (!jobName.matches("[0-9a-zA-Z_]*")) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 1, "作业名只允许包含：数字0-9、小写字符a-z、大写字符A-Z、下划线_。"));
    }
    jobConfig.setJobName(jobName);
    String jobType = getContents(rowCells, 1);
    if (jobType == null || jobType.trim().isEmpty()) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 2, "作业类型必填。"));
    }
    if (JobBriefInfo.JobType.getJobType(jobType).equals(JobBriefInfo.JobType.UNKOWN_JOB)) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 2, "作业类型未知。"));
    }
    if (JobBriefInfo.JobType.getJobType(jobType).equals(JobBriefInfo.JobType.VSHELL) && jobDimensionService.isNewSaturn("1.1.2") != 2) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 2, "Shell消息作业不能导入到包含1.1.2以下版本Executor所在的域。"));
    }
    jobConfig.setJobType(jobType);
    String jobClass = getContents(rowCells, 2);
    if (jobType.equals(JobBriefInfo.JobType.JAVA_JOB.name()) || jobType.equals(JobBriefInfo.JobType.MSG_JOB.name())) {
        if (jobClass == null || jobClass.trim().isEmpty()) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 3, "对于JAVA/MSG作业，作业实现类必填。"));
        }
    }
    jobConfig.setJobClass(jobClass);
    String cron = getContents(rowCells, 3);
    if (jobType.equals(JobBriefInfo.JobType.JAVA_JOB.name()) || jobType.equals(JobBriefInfo.JobType.SHELL_JOB.name())) {
        if (cron == null || cron.trim().isEmpty()) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 4, "对于JAVA/SHELL作业，cron表达式必填。"));
        }
        cron = cron.trim();
        try {
            CronExpression.validateExpression(cron);
        } catch (ParseException e) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 4, "cron表达式语法有误，" + e.toString()));
        }
    } else {
        // 其他类型的不需要持久化保存cron表达式
        cron = "";
    }
    jobConfig.setCron(cron);
    jobConfig.setDescription(getContents(rowCells, 4));
    jobConfig.setLocalMode(Boolean.valueOf(getContents(rowCells, 5)));
    int shardingTotalCount = 1;
    if (jobConfig.getLocalMode()) {
        jobConfig.setShardingTotalCount(shardingTotalCount);
    } else {
        String tmp = getContents(rowCells, 6);
        if (tmp != null) {
            try {
                shardingTotalCount = Integer.parseInt(tmp);
            } catch (NumberFormatException e) {
                throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 7, "分片数有误，" + e.toString()));
            }
        } else {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 7, "分片数必填"));
        }
        if (shardingTotalCount < 1) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 7, "分片数不能小于1"));
        }
        jobConfig.setShardingTotalCount(shardingTotalCount);
    }
    int timeoutSeconds = 0;
    try {
        String tmp = getContents(rowCells, 7);
        if (tmp != null && !tmp.trim().isEmpty()) {
            timeoutSeconds = Integer.parseInt(tmp.trim());
        }
    } catch (NumberFormatException e) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 8, "超时（Kill线程/进程）时间有误，" + e.toString()));
    }
    jobConfig.setTimeoutSeconds(timeoutSeconds);
    jobConfig.setJobParameter(getContents(rowCells, 8));
    String shardingItemParameters = getContents(rowCells, 9);
    if (jobConfig.getLocalMode()) {
        if (shardingItemParameters == null) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 10, "对于本地模式作业，分片参数必填。"));
        } else {
            String[] split = shardingItemParameters.split(",");
            boolean includeXing = false;
            for (String tmp : split) {
                String[] split2 = tmp.split("=");
                if ("*".equalsIgnoreCase(split2[0].trim())) {
                    includeXing = true;
                    break;
                }
            }
            if (!includeXing) {
                throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 10, "对于本地模式作业，分片参数必须包含如*=xx。"));
            }
        }
    } else if (shardingTotalCount > 0) {
        if (shardingItemParameters == null || shardingItemParameters.trim().isEmpty() || shardingItemParameters.split(",").length < shardingTotalCount) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 10, "分片参数不能小于分片总数。"));
        }
    }
    jobConfig.setShardingItemParameters(shardingItemParameters);
    jobConfig.setQueueName(getContents(rowCells, 10));
    jobConfig.setChannelName(getContents(rowCells, 11));
    jobConfig.setPreferList(getContents(rowCells, 12));
    jobConfig.setUseDispreferList(!Boolean.valueOf(getContents(rowCells, 13)));
    int processCountIntervalSeconds = 300;
    try {
        String tmp = getContents(rowCells, 14);
        if (tmp != null && !tmp.trim().isEmpty()) {
            processCountIntervalSeconds = Integer.parseInt(tmp.trim());
        }
    } catch (NumberFormatException e) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 15, "统计处理数据量的间隔秒数有误，" + e.toString()));
    }
    jobConfig.setProcessCountIntervalSeconds(processCountIntervalSeconds);
    int loadLevel = 1;
    try {
        String tmp = getContents(rowCells, 15);
        if (tmp != null && !tmp.trim().isEmpty()) {
            loadLevel = Integer.parseInt(tmp.trim());
        }
    } catch (NumberFormatException e) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 16, "负荷有误，" + e.toString()));
    }
    jobConfig.setLoadLevel(loadLevel);
    jobConfig.setShowNormalLog(Boolean.valueOf(getContents(rowCells, 16)));
    jobConfig.setPausePeriodDate(getContents(rowCells, 17));
    jobConfig.setPausePeriodTime(getContents(rowCells, 18));
    jobConfig.setUseSerial(Boolean.valueOf(getContents(rowCells, 19)));
    int jobDegree = 0;
    try {
        String tmp = getContents(rowCells, 20);
        if (tmp != null && !tmp.trim().isEmpty()) {
            jobDegree = Integer.parseInt(tmp.trim());
        }
    } catch (NumberFormatException e) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 21, "作业重要等级有误，" + e.toString()));
    }
    jobConfig.setJobDegree(jobDegree);
    // 第21列，上报运行状态失效，由算法决定是否上报，看下面setEnabledReport时的逻辑
    String jobMode = getContents(rowCells, 22);
    if (jobMode != null && jobMode.startsWith(JobMode.SYSTEM_PREFIX)) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 23, "作业模式有误，不能添加系统作业"));
    }
    jobConfig.setJobMode(jobMode);
    String dependencies = getContents(rowCells, 23);
    ;
    if (dependencies != null && !dependencies.matches("[0-9a-zA-Z_,]*")) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 24, "依赖的作业只允许包含：数字0-9、小写字符a-z、大写字符A-Z、下划线_、英文逗号,"));
    }
    jobConfig.setDependencies(dependencies);
    jobConfig.setGroups(getContents(rowCells, 24));
    int timeout4AlarmSeconds = 0;
    try {
        String tmp = getContents(rowCells, 25);
        if (tmp != null && !tmp.trim().isEmpty()) {
            timeout4AlarmSeconds = Integer.parseInt(tmp.trim());
        }
    } catch (NumberFormatException e) {
        throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 26, "超时（告警）时间有误，" + e.toString()));
    }
    jobConfig.setTimeout4AlarmSeconds(timeout4AlarmSeconds);
    String timeZone = getContents(rowCells, 26);
    if (timeZone == null || timeZone.trim().length() == 0) {
        timeZone = SaturnConstants.TIME_ZONE_ID_DEFAULT;
    } else {
        timeZone = timeZone.trim();
        if (!SaturnConstants.TIME_ZONE_IDS.contains(timeZone)) {
            throw new SaturnJobConsoleException(createExceptionMessage(sheetNumber, rowNumber, 27, "时区有误"));
        }
    }
    jobConfig.setTimeZone(timeZone);
    return jobConfig;
}

Example 47 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class JobOperationRestApiController method constructJobConfig.

private JobConfig constructJobConfig(String namespace, Map<String, Object> reqParams) throws SaturnJobConsoleException {
    checkMissingParameter("namespace", namespace);
    if (!reqParams.containsKey("jobConfig")) {
        throw new SaturnJobConsoleHttpException(HttpStatus.BAD_REQUEST.value(), String.format(INVALID_REQUEST_MSG, "jobConfig", "cannot be blank"));
    }
    JobConfig jobConfig = new JobConfig();
    Map<String, Object> configParams = (Map<String, Object>) reqParams.get("jobConfig");
    jobConfig.setJobName(checkAndGetParametersValueAsString(reqParams, "jobName", true));
    jobConfig.setDescription(checkAndGetParametersValueAsString(reqParams, "description", false));
    jobConfig.setChannelName(checkAndGetParametersValueAsString(configParams, "channelName", false));
    jobConfig.setCron(checkAndGetParametersValueAsString(configParams, "cron", false));
    jobConfig.setJobClass(checkAndGetParametersValueAsString(configParams, "jobClass", false));
    jobConfig.setJobParameter(checkAndGetParametersValueAsString(configParams, "jobParameter", false));
    String jobType = checkAndGetParametersValueAsString(configParams, "jobType", true);
    if (JobType.UNKOWN_JOB.equals(JobType.getJobType(jobType))) {
        throw new SaturnJobConsoleHttpException(HttpStatus.BAD_REQUEST.value(), String.format(INVALID_REQUEST_MSG, "jobType", "is malformed"));
    }
    jobConfig.setJobType(jobType);
    jobConfig.setLoadLevel(checkAndGetParametersValueAsInteger(configParams, "loadLevel", false));
    jobConfig.setLocalMode(checkAndGetParametersValueAsBoolean(configParams, "localMode", false));
    jobConfig.setPausePeriodDate(checkAndGetParametersValueAsString(configParams, "pausePeriodDate", false));
    jobConfig.setPausePeriodTime(checkAndGetParametersValueAsString(configParams, "pausePeriodTime", false));
    jobConfig.setPreferList(checkAndGetParametersValueAsString(configParams, "preferList", false));
    jobConfig.setQueueName(checkAndGetParametersValueAsString(configParams, "queueName", false));
    jobConfig.setShardingItemParameters(checkAndGetParametersValueAsString(configParams, "shardingItemParameters", true));
    jobConfig.setShardingTotalCount(checkAndGetParametersValueAsInteger(configParams, "shardingTotalCount", true));
    jobConfig.setTimeout4AlarmSeconds(checkAndGetParametersValueAsInteger(configParams, "timeout4AlarmSeconds", false));
    jobConfig.setTimeoutSeconds(checkAndGetParametersValueAsInteger(configParams, "timeout4Seconds", false));
    jobConfig.setUseDispreferList(checkAndGetParametersValueAsBoolean(configParams, "useDispreferList", false));
    jobConfig.setUseSerial(checkAndGetParametersValueAsBoolean(configParams, "useSerial", false));
    jobConfig.setJobDegree(checkAndGetParametersValueAsInteger(configParams, "jobDegree", false));
    jobConfig.setDependencies(checkAndGetParametersValueAsString(configParams, "dependencies", false));
    jobConfig.setTimeZone(checkAndGetParametersValueAsString(configParams, "timeZone", false));
    jobConfig.setTimeoutSeconds(checkAndGetParametersValueAsInteger(configParams, "timeoutSeconds", false));
    jobConfig.setProcessCountIntervalSeconds(checkAndGetParametersValueAsInteger(configParams, "processCountIntervalSeconds", false));
    jobConfig.setGroups(checkAndGetParametersValueAsString(configParams, "groups", false));
    jobConfig.setShowNormalLog(checkAndGetParametersValueAsBoolean(configParams, "showNormalLog", false));
    return jobConfig;
}

Example 48 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class RunAtOnceJobIT method test_C_normalTrigger.

/**
 * 作业STOPPING时立即强制终止
 */
@Test
public void test_C_normalTrigger() throws Exception {
    final int shardCount = 3;
    final String jobName = "test_C_normalTrigger";
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        SimpleJavaJob.statusMap.put(key, 0);
    }
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setTimeoutSeconds(0);
    jobConfig.setShardingItemParameters("0=0,1=1,2=2");
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int i = 0; i < shardCount; i++) {
                    String key = jobName + "_" + i;
                    if (SimpleJavaJob.statusMap.get(key) != 1) {
                        return false;
                    }
                }
                return true;
            }
        }, 30);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    SimpleJavaJob.statusMap.clear();
}

Example 49 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ShardingIT method test_G_ContainerWithUseDispreferList.

/**
 * preferList配置了容器资源，并且useDispreferList为true。当该容器有executor在线，则得到分片；当该容器全部executor下线，则其他executor得到分片
 */
@Test
public void test_G_ContainerWithUseDispreferList() throws Exception {
    // 启动一个非容器executor
    Main executor1 = startOneNewExecutorList();
    boolean cleanOld = SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN;
    String taskOld = SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID;
    try {
        String taskId = "test1";
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = true;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskId;
        // 启动一个容器executor
        Main executor2 = startOneNewExecutorList();
        final int shardCount = 2;
        final String jobName = "test_G_ContainerWithUseDispreferList";
        for (int i = 0; i < shardCount; i++) {
            String key = jobName + "_" + i;
            SimpleJavaJob.statusMap.put(key, 0);
        }
        final JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(jobName);
        jobConfig.setCron("9 9 9 9 9 ? 2099");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(shardCount);
        jobConfig.setShardingItemParameters("0=0,1=1");
        jobConfig.setLocalMode(false);
        // 设置preferList为@taskId
        jobConfig.setPreferList("@" + taskId);
        // 设置useDispreferList为true
        jobConfig.setUseDispreferList(true);
        addJob(jobConfig);
        Thread.sleep(1000);
        enableJob(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // executor2获取到0、1分片，executor1获取不到分片
        List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
        assertThat(items).contains(0, 1);
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
        assertThat(items).isEmpty();
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return hasCompletedZnodeForAllShards(jobName, shardCount);
            }
        }, 10);
        // wait running completed
        Thread.sleep(1000);
        // executor2下线
        stopExecutorGracefully(1);
        Thread.sleep(1000L);
        // 等待sharding分片完成
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        // 等待拿走分片
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // executor1仍然获取0、1分片
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
        assertThat(items).contains(0, 1);
        disableJob(jobName);
        Thread.sleep(1000);
        removeJob(jobName);
        stopExecutorListGracefully();
    } finally {
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = cleanOld;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskOld;
    }
}

Example 50 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ShardingIT method test_K_ContainerWithUseDispreferList_ButInvalidTaskId.

/**
 * preferList配置了无效容器资源，并且useDispreferList为true。则非容器资源会得到分片。
 */
@Test
public void test_K_ContainerWithUseDispreferList_ButInvalidTaskId() throws Exception {
    // 启动一个非容器executor
    Main logicExecutor = startOneNewExecutorList();
    boolean cleanOld = SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN;
    String taskOld = SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID;
    try {
        String taskId = "test1";
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = true;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskId;
        // 启动一个容器executor
        Main vdosExecutor = startOneNewExecutorList();
        int shardCount = 2;
        final String jobName = "test_K_ContainerWithUseDispreferList_ButInvalidTaskId";
        for (int i = 0; i < shardCount; i++) {
            String key = jobName + "_" + i;
            SimpleJavaJob.statusMap.put(key, 0);
        }
        JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(jobName);
        jobConfig.setCron("9 9 9 9 9 ? 2099");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(shardCount);
        jobConfig.setShardingItemParameters("0=0,1=1");
        jobConfig.setLocalMode(false);
        // 设置preferList为@hahataskId
        jobConfig.setPreferList("@haha" + taskId);
        // 设置useDispreferList为true
        jobConfig.setUseDispreferList(true);
        addJob(jobConfig);
        Thread.sleep(1000);
        enableJob(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // vdosExecutor获取不到分片，logicExecutor获取到0、1分片
        List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(vdosExecutor.getExecutorName()))));
        assertThat(items).isEmpty();
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(logicExecutor.getExecutorName()))));
        assertThat(items).contains(0, 1);
        // wait running completed
        Thread.sleep(1000);
        // vdosExecutor下线
        stopExecutorGracefully(1);
        Thread.sleep(1000);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // logicExecutor仍然获取0、1分片
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(logicExecutor.getExecutorName()))));
        assertThat(items).contains(0, 1);
        disableJob(jobName);
        Thread.sleep(1000);
        removeJob(jobName);
        stopExecutorListGracefully();
    } finally {
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = cleanOld;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskOld;
    }
}