Examples with JobConfig com.vip.saturn.job.console.domain.JobConfig used on opensource projects

Example 21 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ShardingIT method test_Q_PersistNecessaryTheRightData.

/**
 * https://github.com/vipshop/Saturn/issues/119
 */
@Test
public void test_Q_PersistNecessaryTheRightData() throws Exception {
    // 启动1个executor
    Main executor1 = startOneNewExecutorList();
    Thread.sleep(1000);
    // 启动第一个作业
    Thread.sleep(1000);
    final String jobName = "test_Q_PersistNecessaryTheRightData";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    jobConfig.setUseDispreferList(false);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String jobLeaderShardingNecessaryNodePath = SaturnExecutorsNode.getJobLeaderShardingNecessaryNodePath(jobName);
    String data1 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data1:" + data1);
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // 启动第2个executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data2 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data2:" + data2);
    assertThat(data2.contains(executor2.getExecutorName())).isTrue();
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // wait running completed
    Thread.sleep(1000);
    // offline executor2
    stopExecutorGracefully(1);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data3 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data3:" + data3);
    assertThat(data3.contains(executor2.getExecutorName())).isFalse();
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 22 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ShardingIT method test_B_JobAverage.

@Test
public void test_B_JobAverage() throws Exception {
    if (!AbstractAsyncShardingTask.ENABLE_JOB_BASED_SHARDING) {
        return;
    }
    // 启动第1台executor
    Main executor1 = startOneNewExecutorList();
    // 添加第1个作业
    final String job1 = "test_B_JobAverage_job1";
    {
        final JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(job1);
        jobConfig.setCron("0/1 * * * * ?");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(2);
        jobConfig.setShardingItemParameters("0=0,1=1");
        addJob(jobConfig);
        Thread.sleep(1000);
    }
    // 添加第2个作业
    final String job2 = "test_B_JobAverage_job2";
    {
        final JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(job2);
        jobConfig.setCron("0/1 * * * * ?");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(2);
        jobConfig.setShardingItemParameters("0=0,1=1");
        addJob(jobConfig);
        Thread.sleep(1000);
    }
    // 启用job1
    enableJob(job1);
    Thread.sleep(2000);
    // 校验
    Map<String, List<Integer>> shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(2).contains(0, 1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(2).contains(0, 1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    // 启动第2台executor
    Main executor2 = startOneNewExecutorList();
    // 等待作业执行获取分片
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).isEmpty();
    // 启用job2
    enableJob(job2);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    // 禁用job1
    disableJob(job1);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    // 由于禁用的作业没有被notify，所以这个节点的内容还是原来的
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    // 下线executor1
    stopExecutorGracefully(0);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).isNull();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isNull();
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(2).contains(0, 1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(2).contains(0, 1);
    // 关闭
    removeJob(job1);
    Thread.sleep(1000);
    disableJob(job2);
    Thread.sleep(1000);
    removeJob(job2);
    stopExecutorListGracefully();
}

Example 23 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class LocalModeIT method test_A.

@Test
public void test_A() throws Exception {
    if (!OS.isFamilyUnix()) {
        return;
    }
    startExecutorList(2);
    final String jobName = "test_A";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName("test_A");
    jobConfig.setCron("*/2 * * * * ?");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setProcessCountIntervalSeconds(1);
    jobConfig.setShardingItemParameters("*=sh " + NORMAL_SH_PATH);
    jobConfig.setLocalMode(true);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            for (Main executor : saturnExecutorList) {
                String count = zkGetJobNode(jobName, "servers/" + executor.getExecutorName() + "/processSuccessCount");
                System.out.println("count:" + count + ";executor:" + executor.getExecutorName());
                if (count == null)
                    return false;
                int times = Integer.parseInt(count);
                if (times <= 0)
                    return false;
            }
            return true;
        }
    }, 10);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}

Example 24 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ScriptJobIT method test_B_ForceStop.

/**
 * 作业启用状态，关闭Executor，将强停作业
 */
@Test
public void test_B_ForceStop() throws Exception {
    // bacause ScriptPidUtils.isPidRunning don't support mac
    if (!OS.isFamilyUnix() || OS.isFamilyMac()) {
        return;
    }
    final int shardCount = 3;
    // 避免多个IT同时跑该作业
    final String jobName = "test_B_ForceStop_" + new Random().nextInt(100);
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=sh " + LONG_TIME_SH_PATH + ",1=sh " + LONG_TIME_SH_PATH + ",2=sh " + LONG_TIME_SH_PATH);
    addJob(jobConfig);
    Thread.sleep(1000);
    // 将会删除该作业的一些pid垃圾数据
    startOneNewExecutorList();
    Thread.sleep(1000);
    final String executorName = saturnExecutorList.get(0).getExecutorName();
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(2000);
    // 不优雅退出，直接关闭
    stopExecutor(0);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    long pid = ScriptPidUtils.getFirstPidFromFile(executorName, jobName, "" + j);
                    if (pid > 0 && ScriptPidUtils.isPidRunning(pid)) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}

Example 25 with JobConfig

use of com.vip.saturn.job.console.domain.JobConfig in project Saturn by vipshop.

the class ScriptJobIT method test_C_ReuseItem.

/**
 * 作业禁用状态，关闭Executor，分片进程不强杀。 下次启动Executor，将其正在运行分片，重新持久化分片状态（running节点），并监听其状态（运行完，删除running节点，持久化completed节点）
 */
@Test
public void test_C_ReuseItem() throws Exception {
    // because ScriptPidUtils.isPidRunning don't supoort mac
    if (!OS.isFamilyUnix() || OS.isFamilyMac()) {
        return;
    }
    final int shardCount = 3;
    // 避免多个IT同时跑该作业
    final String jobName = "test_C_ReuseItem" + new Random().nextInt(100);
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=sh " + LONG_TIME_SH_PATH + ",1=sh " + LONG_TIME_SH_PATH + ",2=sh " + LONG_TIME_SH_PATH);
    addJob(jobConfig);
    Thread.sleep(1000);
    // 将会删除该作业的一些pid垃圾数据
    startOneNewExecutorList();
    Thread.sleep(1000);
    final String executorName = saturnExecutorList.get(0).getExecutorName();
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    disableJob(jobName);
    Thread.sleep(1000);
    // 不优雅退出，直接关闭
    stopExecutor(0);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isOnline(executorName);
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    for (int j = 0; j < shardCount; j++) {
        long pid = ScriptPidUtils.getFirstPidFromFile(executorName, jobName, "" + j);
        if (pid < 0 || !ScriptPidUtils.isPidRunning(pid)) {
            fail("item " + j + ", pid " + pid + " should running");
        }
    }
    startOneNewExecutorList();
    Thread.sleep(2000);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    if (!regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getRunningNode(j)))) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    forceStopJob(jobName);
    Thread.sleep(1000);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    if (!regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getCompletedNode(j)))) {
                        return false;
                    }
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    removeJob(jobName);
}