Examples with Main com.vip.saturn.job.executor.Main used on opensource projects

Example 11 with Main

use of com.vip.saturn.job.executor.Main in project Saturn by vipshop.

the class ShardingIT method test_Q_PersistNecessaryTheRightData.

/**
 * https://github.com/vipshop/Saturn/issues/119
 */
@Test
public void test_Q_PersistNecessaryTheRightData() throws Exception {
    // 启动1个executor
    Main executor1 = startOneNewExecutorList();
    Thread.sleep(1000);
    // 启动第一个作业
    Thread.sleep(1000);
    final String jobName = "test_Q_PersistNecessaryTheRightData";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    jobConfig.setUseDispreferList(false);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String jobLeaderShardingNecessaryNodePath = SaturnExecutorsNode.getJobLeaderShardingNecessaryNodePath(jobName);
    String data1 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data1:" + data1);
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // 启动第2个executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data2 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data2:" + data2);
    assertThat(data2.contains(executor2.getExecutorName())).isTrue();
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // wait running completed
    Thread.sleep(1000);
    // offline executor2
    stopExecutorGracefully(1);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data3 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data3:" + data3);
    assertThat(data3.contains(executor2.getExecutorName())).isFalse();
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 12 with Main

use of com.vip.saturn.job.executor.Main in project Saturn by vipshop.

the class ShardingIT method test_B_JobAverage.

@Test
public void test_B_JobAverage() throws Exception {
    if (!AbstractAsyncShardingTask.ENABLE_JOB_BASED_SHARDING) {
        return;
    }
    // 启动第1台executor
    Main executor1 = startOneNewExecutorList();
    // 添加第1个作业
    final String job1 = "test_B_JobAverage_job1";
    {
        final JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(job1);
        jobConfig.setCron("0/1 * * * * ?");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(2);
        jobConfig.setShardingItemParameters("0=0,1=1");
        addJob(jobConfig);
        Thread.sleep(1000);
    }
    // 添加第2个作业
    final String job2 = "test_B_JobAverage_job2";
    {
        final JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(job2);
        jobConfig.setCron("0/1 * * * * ?");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(2);
        jobConfig.setShardingItemParameters("0=0,1=1");
        addJob(jobConfig);
        Thread.sleep(1000);
    }
    // 启用job1
    enableJob(job1);
    Thread.sleep(2000);
    // 校验
    Map<String, List<Integer>> shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(2).contains(0, 1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(2).contains(0, 1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    // 启动第2台executor
    Main executor2 = startOneNewExecutorList();
    // 等待作业执行获取分片
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).isEmpty();
    // 启用job2
    enableJob(job2);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    // 禁用job1
    disableJob(job1);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).isEmpty();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).hasSize(1).contains(0);
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    // 由于禁用的作业没有被notify，所以这个节点的内容还是原来的
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).hasSize(1).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    // 下线executor1
    stopExecutorGracefully(0);
    Thread.sleep(2000);
    // 校验
    shardingItems = namespaceShardingContentService.getShardingItems(job1);
    assertThat(shardingItems.get(executor1.getExecutorName())).isNull();
    assertThat(shardingItems.get(executor2.getExecutorName())).isEmpty();
    shardingItems = namespaceShardingContentService.getShardingItems(job2);
    assertThat(shardingItems.get(executor1.getExecutorName())).isNull();
    assertThat(shardingItems.get(executor2.getExecutorName())).hasSize(2).contains(0, 1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(1).contains(1);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(job2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).hasSize(2).contains(0, 1);
    // 关闭
    removeJob(job1);
    Thread.sleep(1000);
    disableJob(job2);
    Thread.sleep(1000);
    removeJob(job2);
    stopExecutorListGracefully();
}

Example 13 with Main

use of com.vip.saturn.job.executor.Main in project Saturn by vipshop.

the class LocalModeIT method test_A.

@Test
public void test_A() throws Exception {
    if (!OS.isFamilyUnix()) {
        return;
    }
    startExecutorList(2);
    final String jobName = "test_A";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName("test_A");
    jobConfig.setCron("*/2 * * * * ?");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setProcessCountIntervalSeconds(1);
    jobConfig.setShardingItemParameters("*=sh " + NORMAL_SH_PATH);
    jobConfig.setLocalMode(true);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            for (Main executor : saturnExecutorList) {
                String count = zkGetJobNode(jobName, "servers/" + executor.getExecutorName() + "/processSuccessCount");
                System.out.println("count:" + count + ";executor:" + executor.getExecutorName());
                if (count == null)
                    return false;
                int times = Integer.parseInt(count);
                if (times <= 0)
                    return false;
            }
            return true;
        }
    }, 10);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}

Example 14 with Main

use of com.vip.saturn.job.executor.Main in project Saturn by vipshop.

the class RestartAndDumpIT method test.

@Test
public void test() throws Exception {
    if (!OS.isFamilyUnix()) {
        return;
    }
    SystemEnvProperties.VIP_SATURN_ENABLE_EXEC_SCRIPT = true;
    SystemEnvProperties.VIP_SATURN_PRG = new File(PRG).getAbsolutePath();
    SystemEnvProperties.VIP_SATURN_LOG_OUTFILE = new File(OUTFILE).getAbsolutePath();
    try {
        File outfile = new File(OUTFILE);
        if (outfile.exists()) {
            assertThat(outfile.delete()).isTrue();
        }
        Main executor = startOneNewExecutorList();
        regCenter.persist(SaturnExecutorsNode.EXECUTORS_ROOT + "/" + executor.getExecutorName() + "/restart", "");
        Thread.sleep(1000L);
        String content = FileUtils.readFileToString(new File(OUTFILE));
        assertThat(content).isEqualTo("Hebe! Hebe! Hebe!");
        String dumpNodePath = SaturnExecutorsNode.EXECUTORS_ROOT + "/" + executor.getExecutorName() + "/dump";
        regCenter.persist(dumpNodePath, "");
        Thread.sleep(1000L);
        content = FileUtils.readFileToString(new File(OUTFILE));
        assertThat(content).isEqualTo("Hebe! Hebe! Hebe!Dump! Dump! Dump!");
        assertThat(regCenter.isExisted(dumpNodePath)).isFalse();
    } finally {
        SystemEnvProperties.VIP_SATURN_ENABLE_EXEC_SCRIPT = false;
        SystemEnvProperties.VIP_SATURN_PRG = null;
        SystemEnvProperties.VIP_SATURN_LOG_OUTFILE = null;
    }
}

Example 15 with Main

use of com.vip.saturn.job.executor.Main in project Saturn by vipshop.

the class ShardingWithChangingLeaderIT method test_A_StopNamespaceShardingManagerLeader.

@Test
public void test_A_StopNamespaceShardingManagerLeader() throws Exception {
    startSaturnConsoleList(2);
    Thread.sleep(1000);
    final String jobName = "test_A_StopNamespaceShardingManagerLeader";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    addJob(jobConfig);
    Thread.sleep(1000);
    Main executor = startOneNewExecutorList();
    Thread.sleep(1000);
    String hostValue = regCenter.get(SaturnExecutorsNode.LEADER_HOSTNODE_PATH);
    assertThat(hostValue).isNotNull();
    stopSaturnConsole(0);
    Thread.sleep(1000);
    String hostValue2 = regCenter.get(SaturnExecutorsNode.LEADER_HOSTNODE_PATH);
    assertThat(hostValue2).isNotNull().isNotEqualTo(hostValue);
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor.getExecutorName()))));
    assertThat(items).contains(0);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    Thread.sleep(1000);
    stopExecutorListGracefully();
    stopSaturnConsoleList();
}