Examples with FinishCheck com.vip.saturn.it.base.FinishCheck used on opensource projects

Example 11 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_E_PreferListOnly.

@Test
public void test_E_PreferListOnly() throws Exception {
    // 启动第1台executor
    Main executor1 = startOneNewExecutorList();
    int shardCount = 3;
    final String jobName = "test_E_PreferListOnly";
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        SimpleJavaJob.statusMap.put(key, 0);
    }
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("0=0,1=1,2=2");
    jobConfig.setPreferList(executor1.getExecutorName());
    jobConfig.setUseDispreferList(false);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).contains(0, 1, 2);
    // 启动第2台executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).contains(0, 1, 2);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isEmpty();
    // 停第1个executor
    stopExecutorGracefully(0);
    Thread.sleep(1000);
    runAtOnce(jobName);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            if (isNeedSharding(jobName)) {
                return false;
            }
            return true;
        }
    }, 10);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isEmpty();
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    System.out.println(items);
    assertThat(items).isEmpty();
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 12 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_M_UseDispreferList_ButInvalidLogicPreferList.

/**
 * preferList配置了无效物理资源，并且useDispreferList为true。先添加作业，启用作业，再启动容器，再启动物理机。则物理资源会得到分片。
 */
@Test
public void test_M_UseDispreferList_ButInvalidLogicPreferList() throws Exception {
    boolean cleanOld = SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN;
    String taskOld = SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID;
    try {
        int shardCount = 2;
        final String jobName = "test_M_UseDispreferList_ButInvalidLogicPreferList";
        for (int i = 0; i < shardCount; i++) {
            String key = jobName + "_" + i;
            SimpleJavaJob.statusMap.put(key, 0);
        }
        JobConfig jobConfig = new JobConfig();
        jobConfig.setJobName(jobName);
        jobConfig.setCron("9 9 9 9 9 ? 2099");
        jobConfig.setJobType(JobType.JAVA_JOB.toString());
        jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
        jobConfig.setShardingTotalCount(shardCount);
        jobConfig.setShardingItemParameters("0=0,1=1");
        jobConfig.setLocalMode(false);
        // 设置preferList为@haha
        jobConfig.setPreferList("haha");
        // 设置useDispreferList为true
        jobConfig.setUseDispreferList(true);
        addJob(jobConfig);
        Thread.sleep(1000);
        enableJob(jobName);
        // 启动一个容器executor
        String taskId = "test1";
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = true;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskId;
        Main vdosExecutor = startOneNewExecutorList();
        // 启动一个非容器executor
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = false;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = null;
        Main logicExecutor = startOneNewExecutorList();
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // vdosExecutor获取不到分片，logicExecutor获取到0、1分片
        List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(vdosExecutor.getExecutorName()))));
        assertThat(items).isEmpty();
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(logicExecutor.getExecutorName()))));
        assertThat(items).contains(0, 1);
        // wait running completed
        Thread.sleep(1000);
        // vdosExecutor下线
        stopExecutorGracefully(0);
        Thread.sleep(1000L);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return isNeedSharding(jobName);
            }
        }, 10);
        runAtOnce(jobName);
        waitForFinish(new FinishCheck() {

            @Override
            public boolean isOk() {
                return !isNeedSharding(jobName);
            }
        }, 10);
        // logicExecutor仍然获取0、1分片
        items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(logicExecutor.getExecutorName()))));
        assertThat(items).contains(0, 1);
        disableJob(jobName);
        Thread.sleep(1000);
        removeJob(jobName);
        stopExecutorListGracefully();
    } finally {
        SystemEnvProperties.VIP_SATURN_EXECUTOR_CLEAN = cleanOld;
        SystemEnvProperties.VIP_SATURN_CONTAINER_DEPLOYMENT_ID = taskOld;
    }
}

Example 13 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_F_LocalModeWithPreferListAndUseDispreferList.

/**
 * 本地模式作业，配置了preferList，即使配置useDispreferList为true，但是仍然只有preferList能得到该作业分片。即useDispreferList对本地模式作业不起作用。
 */
@Test
public void test_F_LocalModeWithPreferListAndUseDispreferList() throws Exception {
    Main executor1 = startOneNewExecutorList();
    Main executor2 = startOneNewExecutorList();
    int shardCount = 2;
    final String jobName = "test_F_LocalModeWithPreferListAndUseDispreferList";
    for (int i = 0; i < shardCount; i++) {
        String key = jobName + "_" + i;
        SimpleJavaJob.statusMap.put(key, 0);
    }
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(shardCount);
    jobConfig.setShardingItemParameters("*=0");
    jobConfig.setLocalMode(true);
    // 设置preferList为executor2
    jobConfig.setPreferList(executor2.getExecutorName());
    // 设置useDispreferList为true
    jobConfig.setUseDispreferList(true);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // executor2获取到0分片，executor1获取不到分片
    List<Integer> items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor2.getExecutorName()))));
    assertThat(items).contains(0);
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    // wait running completed
    Thread.sleep(1000);
    // executor2下线
    stopExecutorGracefully(1);
    Thread.sleep(1000L);
    // 等待sharding分片完成
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    runAtOnce(jobName);
    // 等待拿走分片
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // executor1仍然获取不到分片
    items = ItemUtils.toItemList(regCenter.getDirectly(JobNodePath.getNodeFullPath(jobName, ShardingNode.getShardingNode(executor1.getExecutorName()))));
    assertThat(items).isEmpty();
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 14 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class ShardingIT method test_Q_PersistNecessaryTheRightData.

/**
 * https://github.com/vipshop/Saturn/issues/119
 */
@Test
public void test_Q_PersistNecessaryTheRightData() throws Exception {
    // 启动1个executor
    Main executor1 = startOneNewExecutorList();
    Thread.sleep(1000);
    // 启动第一个作业
    Thread.sleep(1000);
    final String jobName = "test_Q_PersistNecessaryTheRightData";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName(jobName);
    jobConfig.setCron("9 9 9 9 9 ? 2099");
    jobConfig.setJobType(JobType.JAVA_JOB.toString());
    jobConfig.setJobClass(SimpleJavaJob.class.getCanonicalName());
    jobConfig.setShardingTotalCount(1);
    jobConfig.setShardingItemParameters("0=0");
    jobConfig.setUseDispreferList(false);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String jobLeaderShardingNecessaryNodePath = SaturnExecutorsNode.getJobLeaderShardingNecessaryNodePath(jobName);
    String data1 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data1:" + data1);
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // 启动第2个executor
    Main executor2 = startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data2 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data2:" + data2);
    assertThat(data2.contains(executor2.getExecutorName())).isTrue();
    runAtOnce(jobName);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return !isNeedSharding(jobName);
        }
    }, 10);
    // wait running completed
    Thread.sleep(1000);
    // offline executor2
    stopExecutorGracefully(1);
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            return isNeedSharding(jobName);
        }
    }, 10);
    String data3 = regCenter.getDirectly(jobLeaderShardingNecessaryNodePath);
    System.out.println("data3:" + data3);
    assertThat(data3.contains(executor2.getExecutorName())).isFalse();
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
    stopExecutorListGracefully();
}

Example 15 with FinishCheck

use of com.vip.saturn.it.base.FinishCheck in project Saturn by vipshop.

the class LocalModeIT method test_A.

@Test
public void test_A() throws Exception {
    if (!OS.isFamilyUnix()) {
        return;
    }
    startExecutorList(2);
    final String jobName = "test_A";
    JobConfig jobConfig = new JobConfig();
    jobConfig.setJobName("test_A");
    jobConfig.setCron("*/2 * * * * ?");
    jobConfig.setJobType(JobType.SHELL_JOB.toString());
    jobConfig.setProcessCountIntervalSeconds(1);
    jobConfig.setShardingItemParameters("*=sh " + NORMAL_SH_PATH);
    jobConfig.setLocalMode(true);
    addJob(jobConfig);
    Thread.sleep(1000);
    enableJob(jobName);
    Thread.sleep(1000);
    startOneNewExecutorList();
    Thread.sleep(1000);
    waitForFinish(new FinishCheck() {

        @Override
        public boolean isOk() {
            for (Main executor : saturnExecutorList) {
                String count = zkGetJobNode(jobName, "servers/" + executor.getExecutorName() + "/processSuccessCount");
                System.out.println("count:" + count + ";executor:" + executor.getExecutorName());
                if (count == null)
                    return false;
                int times = Integer.parseInt(count);
                if (times <= 0)
                    return false;
            }
            return true;
        }
    }, 10);
    disableJob(jobName);
    Thread.sleep(1000);
    removeJob(jobName);
}