Examples with FinishCheck com.vip.saturn.it.base.FinishCheck used on opensource projects

Example 36 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class FailoverIT method failoverWithDisabled.

/**
 * 在failover执行之前禁用的作业重新启用后不应该继续上次的failover流程
 */
private void failoverWithDisabled(final int shardCount, final String jobName, final int disableTime) throws Exception {
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        LongtimeJavaJob.JobStatus status = new LongtimeJavaJob.JobStatus();
        status.runningCount = 0;
        status.sleepSeconds = 20;
        status.finished = false;
        status.timeout = false;
        LongtimeJavaJob.statusMap.put(key, status);
    }
    // 1 新建一个执行时间为10S的作业，它只能手工触发
    final JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(LongtimeJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=0,1=1,2=2");
    addJob(jobConfig);
    Thread.sleep(1000);
    // 2 启动作业并立刻执行一次
    enableJob(jobName);
    Thread.sleep(2000);
    runAtOnce(jobName);
    // 3 保证全部作业分片正在运行中
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int j = 0; j < shardCount; j++) {
                    if (!regCenter.isExisted(JobNodePath.getNodeFullPath(jobName, ExecutionNode.getRunningNode(j)))) {
                        return false;
                    }
                }
                return true;
            }
        }, 6);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    Thread.sleep(2000);
    final String firstExecutorName = saturnExecutorList.get(0).getExecutorName();
    final List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(firstExecutorName))));
    final String secondExecutorName = saturnExecutorList.get(1).getExecutorName();
    final List<Integer> items2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(secondExecutorName))));
    if (disableTime == 1) {
        disableJob(jobName);
    }
    // 4 停止第一个executor，在该executor上运行的分片会失败转移
    stopExecutor(0);
    System.out.println("items:" + items);
    // 5 直到第一个Executor完全下线
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                if (isOnline(firstExecutorName)) {
                    // 判断该Executor是否在线
                    return false;
                }
                return true;
            }
        }, 20);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    // 6 检查停止的executor 上面的分片是否已经被KILL
    for (Integer item : items) {
        String key = jobName + "_" + item;
        LongtimeJavaJob.JobStatus status = LongtimeJavaJob.statusMap.get(key);
        if (!status.finished || status.killed == 0) {
            fail("should finish and killed");
        }
        status.runningCount = 0;
    }
    // 7 检查运行executor2上的分片都正在运行，而且runningCount为0
    for (Integer item : items2) {
        String key = jobName + "_" + item;
        LongtimeJavaJob.JobStatus status = LongtimeJavaJob.statusMap.get(key);
        if (status.finished || status.killed > 0 || status.timeout) {
            fail("should running");
        }
        if (status.runningCount != 0) {
            fail("runningCount should be 0");
        }
    }
    // 8 禁用作业
    if (disableTime == 2) {
        disableJob(jobName);
    }
    // 9 等待executor2分片运行完
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (Integer item : items2) {
                    String key = jobName + "_" + item;
                    LongtimeJavaJob.JobStatus status = LongtimeJavaJob.statusMap.get(key);
                    if (!status.finished) {
                        return false;
                    }
                }
                return true;
            }
        }, 20);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    // 10 检测无failover信息
    assertThat(noFailoverItems(jobName));
    for (Integer item : items) {
        assertThat(isFailoverAssigned(jobName, item)).isEqualTo(false);
    }
    // 11 检测只executor2的分片只运行了一次
    Thread.sleep(2000);
    for (Integer item : items2) {
        String key = jobName + "_" + item;
        LongtimeJavaJob.JobStatus status = LongtimeJavaJob.statusMap.get(key);
        if (status.runningCount != 1) {
            fail("runningCount should be 1");
        }
    }
    removeJob(jobName);
    Thread.sleep(2000);
    LongtimeJavaJob.statusMap.clear();
}

Example 37 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_A_JAVA.

@Test
public void test_A_JAVA() throws Exception {
    int shardCount = 3;
    final String jobName = "test_A_JAVA";
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        SimpleJavaJob.statusMap.put(key, 0);
    }
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=0,1=1,2=2");
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    // 启动第1台executor
    Main executor1 = startOneNewExecutorList();
    runAtOnce(jobName);
    Thread.sleep(1000);
    assertThat(regCenter.getDirectly(SaturnExecutorsNode.SHARDING_COUNT_PATH)).isEqualTo("4");
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).contains(0, 1, 2);
    // 启动第2台executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isNotEmpty();
    System.out.println(items);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isNotEmpty();
    // 启动第3台executor
    Main executor3 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor3.getExecutorName()))));
    System.out.println(items);
    assertThat(items).hasSize(1);
    // 停第1个executor
    stopExecutorGracefully(0);
    Thread.sleep(1000);
    assertThat(regCenter.getDirectly(SaturnExecutorsNode.SHARDING_COUNT_PATH)).isEqualTo("10");
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isEmpty();
    // 停第2个executor
    stopExecutorGracefully(1);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isEmpty();
    // 分片全部落到第3个executor
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor3.getExecutorName()))));
    assertThat(items).contains(0, 1, 2);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 38 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingWithLoadIT method multiJobSharding.

public void multiJobSharding(String preferListJob3) throws Exception {
    // 作业1，负荷为1,分片数为2
    final String jobName1 = "JOB_LOAD1_SHARDING2";
    final JobConfig jobConfig1 = new JobConfig();
    jobConfig1.setJobName(jobName1);
    jobConfig1.setCron("9 9 9 9 9 ? 2099");
    jobConfig1.setJobType(JobType.JAVA_JOB.toString());
    jobConfig1.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig1.setLoadLevel(1);
    jobConfig1.setShardingItemParameters("0=0,1=1,2=2");
    addSimpleJavaJob(jobName1, 2, jobConfig1);
    // 作业2，负荷为2,分片数为2
    final String jobName2 = "JOB_LOAD2_SHARDING2";
    final JobConfig jobConfig2 = new JobConfig();
    jobConfig2.setJobName(jobName2);
    jobConfig2.setCron("9 9 9 9 9 ? 2099");
    jobConfig2.setJobType(JobType.JAVA_JOB.toString());
    jobConfig2.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig2.setLoadLevel(2);
    jobConfig2.setShardingItemParameters("0=0,1=1");
    addSimpleJavaJob(jobName2, 2, jobConfig2);
    // 作业3，负荷为1,分片数为3
    final String jobName3 = "JOB_LOAD1_SHARDING3";
    final JobConfig jobConfig3 = new JobConfig();
    jobConfig3.setJobName(jobName3);
    jobConfig3.setCron("9 9 9 9 9 ? 2099");
    jobConfig3.setJobType(JobType.JAVA_JOB.toString());
    jobConfig3.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig3.setLoadLevel(1);
    jobConfig3.setShardingItemParameters("0=0,1=1,2=2");
    // 没有preferlist or 设置preferlist1,2
    if (null != preferListJob3) {
        jobConfig3.setPreferList(preferListJob3);
    }
    addSimpleJavaJob(jobName3, 3, jobConfig3);
    Thread.sleep(1000);
    // 启用作业1,2,3
    enableJob(jobName1);
    enableJob(jobName2);
    enableJob(jobName3);
    Thread.sleep(1000);
    // 启动第1台executor
    Main executor1 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName1);
    runAtOnce(jobName2);
    runAtOnce(jobName3);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName1) || isNeedSharding(jobName2) || isNeedSharding(jobName3)) {
                return false;
            }
            return true;
        }
    }, 30);
    List<Integer> itemsJob1Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(itemsJob1Exe1).contains(0, 1);
    List<Integer> itemsJob2Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(itemsJob2Exe1).contains(0, 1);
    List<Integer> itemsJob3Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(itemsJob3Exe1).contains(0, 1, 2);
    // 启动第2台executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName1);
    runAtOnce(jobName2);
    runAtOnce(jobName3);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName1) || isNeedSharding(jobName2) || isNeedSharding(jobName3)) {
                return false;
            }
            return true;
        }
    }, 60);
    // 大负荷作业Job2分片都到节点2
    List<Integer> itemsJob2Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println("job2 at exe2 :" + itemsJob2Exe2);
    List<Integer> itemsJob1Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println("job1 at exe2:" + itemsJob1Exe2);
    List<Integer> itemsJob3Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println("job3 at exe2:" + itemsJob3Exe2);
    int totalLoadOfExe2 = itemsJob2Exe2.size() * 2 + itemsJob1Exe2.size() + itemsJob3Exe2.size();
    assertEquals("total load of exe2 not equal", 4, totalLoadOfExe2);
    // 启动第3台executor
    Main executor3 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName1);
    runAtOnce(jobName2);
    runAtOnce(jobName3);
    Thread.sleep(1000);
    List<Integer> itemsJob1Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor3.getExecutorName()))));
    List<Integer> itemsJob2Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor3.getExecutorName()))));
    List<Integer> itemsJob3Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor3.getExecutorName()))));
    System.out.println("itemsJob1Exe3 : " + itemsJob1Exe3);
    System.out.println("itemsJob2Exe3 : " + itemsJob2Exe3);
    System.out.println("itemsJob3Exe3 : " + itemsJob3Exe3);
    // 节点3应该有分片
    Assert.assertFalse("The exe3 has no sharding. ", itemsJob1Exe3.isEmpty() && itemsJob2Exe3.isEmpty() && itemsJob3Exe3.isEmpty());
    // 当job3设置优先节点的情况下
    if (null != preferListJob3) {
        // 节点3不应该有job3的分片
        Assert.assertTrue("The exe3 should have no sharding of job3. ", itemsJob3Exe3.isEmpty());
    }
    // 停节点1前先获取节点1上所有作业分片
    itemsJob1Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    itemsJob2Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    itemsJob3Exe1 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    // 停第1个executor
    stopExecutorGracefully(0);
    Thread.sleep(1000);
    runAtOnce(jobName1);
    runAtOnce(jobName2);
    runAtOnce(jobName3);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName1) || isNeedSharding(jobName2) || isNeedSharding(jobName3)) {
                return false;
            }
            return true;
        }
    }, 10);
    if (!itemsJob1Exe1.isEmpty()) {
        itemsJob1Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor2.getExecutorName()))));
        itemsJob1Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName1, ShardingNode.getShardingNode(executor3.getExecutorName()))));
        for (Integer i : itemsJob1Exe1) {
            Assert.assertTrue("the sharding of exe1 should be shift to exe2 or exe3", itemsJob1Exe2.contains(i) || itemsJob1Exe3.contains(i));
        }
    }
    if (!itemsJob2Exe1.isEmpty()) {
        itemsJob2Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor2.getExecutorName()))));
        itemsJob2Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName2, ShardingNode.getShardingNode(executor3.getExecutorName()))));
        for (Integer i : itemsJob2Exe1) {
            Assert.assertTrue("the sharding of exe1 should be shift to exe2 or exe3", itemsJob2Exe2.contains(i) || itemsJob2Exe3.contains(i));
        }
    }
    if (!itemsJob3Exe1.isEmpty()) {
        itemsJob3Exe2 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor2.getExecutorName()))));
        itemsJob3Exe3 = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName3, ShardingNode.getShardingNode(executor3.getExecutorName()))));
        for (Integer i : itemsJob3Exe1) {
            Assert.assertTrue("the sharding of exe1 should be shift to exe2 or exe3", itemsJob3Exe2.contains(i) || itemsJob3Exe3.contains(i));
        }
    }
    disableJob(jobName1);
    disableJob(jobName2);
    disableJob(jobName3);
    Thread.sleep(1000);
    removeJob(jobName1);
    removeJob(jobName2);
    removeJob(jobName3);
    Thread.sleep(1000);
    stopExecutorListGracefully();
}

Example 39 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class UpdateConfigIT method updateShowNormalLog.

/**
 * 作业配置正常显示日志时，zk的jobLog会有执行日志的IT
 */
@Test
public void updateShowNormalLog() throws Exception {
    final int shardCount = 1;
    final String jobName = "updateShowNormalLog";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setTimeZone(TimeZone.getDefault().getID());
    jobConfig.setCron("*/1 * * * * ?");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=0");
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(2 * 1000);
    for (int i = 0; i < shardCount; i++) {
        String path = JobNodePath.getNodeFullPath(jobName, ExecutionNode.getJobLog(i));
        assertThat(regCenter.isExisted(path)).isEqualTo(false);
    }
    disableJob(jobName);
    Thread.sleep(2 * 1000);
    zkUpdateJobNode(jobName, "config/showNormalLog", "true");
    Thread.sleep(1 * 1000);
    enableJob(jobName);
    Thread.sleep(4000);
    doReport(jobName);
    try {
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                for (int i = 0; i < shardCount; i++) {
                    String path = JobNodePath.getNodeFullPath(jobName, ExecutionNode.getJobLog(i));
                    if (regCenter.isExisted(path)) {
                        assertThat(regCenter.getDirectly(path)).isNotEmpty();
                        continue;
                    }
                    return false;
                }
                return true;
            }
        }, 10);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    Thread.sleep(2000);
}

Example 40 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class LocalModeIT method test_B.

@Test
public void test_B() throws Exception {
    startExecutorList(2);
    int shardCount = 3;
    final String jobName = "test_B";
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        SimpleJavaJob.statusMap.put(key, 0);
    }
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("*/1 * * * * ?");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setProcessCountIntervalSeconds(1);
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingItemParameters("*=0");
    jobConfig.setLocalMode(true);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            for (Main executor : saturnExecutorList) {
                String count = zkGetJobNode(jobName, "servers/" + executor.getExecutorName() + "/processSuccessCount");
                System.out.println("count:" + count + ";executor:" + executor.getExecutorName());
                if (count == null)
                    return false;
                int times = Integer.parseInt(count);
                if (times <= 0)
                    return false;
            }
            return true;
        }
    }, 60);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}